AsPartitionProcessing

This commit is contained in:
christianwade 2016-12-01 13:39:44 -08:00
commit 1395870498
22 changed files with 7893 additions and 0 deletions

63
.gitattributes vendored Normal file
View File

@ -0,0 +1,63 @@
###############################################################################
# Set default behavior to automatically normalize line endings.
###############################################################################
* text=auto
###############################################################################
# Set default behavior for command prompt diff.
#
# This is need for earlier builds of msysgit that does not have it on by
# default for csharp files.
# Note: This is only used by command line
###############################################################################
#*.cs diff=csharp
###############################################################################
# Set the merge driver for project and solution files
#
# Merging from the command prompt will add diff markers to the files if there
# are conflicts (Merging from VS is not affected by the settings below, in VS
# the diff markers are never inserted). Diff markers may cause the following
# file extensions to fail to load in VS. An alternative would be to treat
# these files as binary and thus will always conflict and require user
# intervention with every merge. To do so, just uncomment the entries below
###############################################################################
#*.sln merge=binary
#*.csproj merge=binary
#*.vbproj merge=binary
#*.vcxproj merge=binary
#*.vcproj merge=binary
#*.dbproj merge=binary
#*.fsproj merge=binary
#*.lsproj merge=binary
#*.wixproj merge=binary
#*.modelproj merge=binary
#*.sqlproj merge=binary
#*.wwaproj merge=binary
###############################################################################
# behavior for image files
#
# image files are treated as binary by default.
###############################################################################
#*.jpg binary
#*.png binary
#*.gif binary
###############################################################################
# diff behavior for common document formats
#
# Convert binary document formats to text before diffing them. This feature
# is only available from the command line. Turn it on by uncommenting the
# entries below.
###############################################################################
#*.doc diff=astextplain
#*.DOC diff=astextplain
#*.docx diff=astextplain
#*.DOCX diff=astextplain
#*.dot diff=astextplain
#*.DOT diff=astextplain
#*.pdf diff=astextplain
#*.PDF diff=astextplain
#*.rtf diff=astextplain
#*.RTF diff=astextplain

242
.gitignore vendored Normal file
View File

@ -0,0 +1,242 @@
## Ignore Visual Studio temporary files, build results, and
## files generated by popular Visual Studio add-ons.
# User-specific files
*.suo
*.user
*.userosscache
*.sln.docstates
# User-specific files (MonoDevelop/Xamarin Studio)
*.userprefs
# Build results
[Dd]ebug/
[Dd]ebugPublic/
[Rr]elease/
[Rr]eleases/
[Xx]64/
[Xx]86/
[Bb]uild/
bld/
[Bb]in/
[Oo]bj/
# Visual Studio 2015 cache/options directory
.vs/
# Uncomment if you have tasks that create the project's static files in wwwroot
#wwwroot/
# MSTest test Results
[Tt]est[Rr]esult*/
[Bb]uild[Ll]og.*
# NUNIT
*.VisualState.xml
TestResult.xml
# Build Results of an ATL Project
[Dd]ebugPS/
[Rr]eleasePS/
dlldata.c
# DNX
project.lock.json
artifacts/
*_i.c
*_p.c
*_i.h
*.ilk
*.meta
*.obj
*.pch
*.pdb
*.pgc
*.pgd
*.rsp
*.sbr
*.tlb
*.tli
*.tlh
*.tmp
*.tmp_proj
*.log
*.vspscc
*.vssscc
.builds
*.pidb
*.svclog
*.scc
# Chutzpah Test files
_Chutzpah*
# Visual C++ cache files
ipch/
*.aps
*.ncb
*.opendb
*.opensdf
*.sdf
*.cachefile
*.VC.db
# Visual Studio profiler
*.psess
*.vsp
*.vspx
*.sap
# TFS 2012 Local Workspace
$tf/
# Guidance Automation Toolkit
*.gpState
# ReSharper is a .NET coding add-in
_ReSharper*/
*.[Rr]e[Ss]harper
*.DotSettings.user
# JustCode is a .NET coding add-in
.JustCode
# TeamCity is a build add-in
_TeamCity*
# DotCover is a Code Coverage Tool
*.dotCover
# NCrunch
_NCrunch_*
.*crunch*.local.xml
nCrunchTemp_*
# MightyMoose
*.mm.*
AutoTest.Net/
# Web workbench (sass)
.sass-cache/
# Installshield output folder
[Ee]xpress/
# DocProject is a documentation generator add-in
DocProject/buildhelp/
DocProject/Help/*.HxT
DocProject/Help/*.HxC
DocProject/Help/*.hhc
DocProject/Help/*.hhk
DocProject/Help/*.hhp
DocProject/Help/Html2
DocProject/Help/html
# Click-Once directory
publish/
# Publish Web Output
*.[Pp]ublish.xml
*.azurePubxml
# TODO: Un-comment the next line if you do not want to checkin
# your web deploy settings because they may include unencrypted
# passwords
#*.pubxml
*.publishproj
# NuGet Packages
*.nupkg
# The packages folder can be ignored because of Package Restore
**/packages/*
# except build/, which is used as an MSBuild target.
!**/packages/build/
# Uncomment if necessary however generally it will be regenerated when needed
#!**/packages/repositories.config
# NuGet v3's project.json files produces more ignoreable files
*.nuget.props
*.nuget.targets
# Microsoft Azure Build Output
csx/
*.build.csdef
# Microsoft Azure Emulator
ecf/
rcf/
# Windows Store app package directory
AppPackages/
BundleArtifacts/
# Visual Studio cache files
# files ending in .cache can be ignored
*.[Cc]ache
# but keep track of directories ending in .cache
!*.[Cc]ache/
# Others
ClientBin/
[Ss]tyle[Cc]op.*
~$*
*~
*.dbmdl
*.dbproj.schemaview
*.pfx
*.publishsettings
node_modules/
orleans.codegen.cs
# RIA/Silverlight projects
Generated_Code/
# Backup & report files from converting an old project file
# to a newer Visual Studio version. Backup files are not needed,
# because we have git ;-)
_UpgradeReport_Files/
Backup*/
UpgradeLog*.XML
UpgradeLog*.htm
# SQL Server files
*.mdf
*.ldf
# Business Intelligence projects
*.rdl.data
*.bim.layout
*.bim_*.settings
# Microsoft Fakes
FakesAssemblies/
# GhostDoc plugin setting file
*.GhostDoc.xml
# Node.js Tools for Visual Studio
.ntvs_analysis.dat
# Visual Studio 6 build log
*.plg
# Visual Studio 6 workspace options file
*.opt
# Visual Studio LightSwitch build output
**/*.HTMLClient/GeneratedArtifacts
**/*.DesktopClient/GeneratedArtifacts
**/*.DesktopClient/ModelManifest.xml
**/*.Server/GeneratedArtifacts
**/*.Server/ModelManifest.xml
_Pvt_Extensions
# LightSwitch generated files
GeneratedArtifacts/
ModelManifest.xml
# Paket dependency manager
.paket/paket.exe
# FAKE - F# Make
.fake/

Binary file not shown.

View File

@ -0,0 +1,42 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Development</Configuration>
<SchemaVersion>2.0</SchemaVersion>
<ProjectGuid>{7274efcc-e7df-40ad-83df-1dfd5b954e67}</ProjectGuid>
<OutputType>Exe</OutputType>
<RootNamespace>MyRootNamespace</RootNamespace>
<AssemblyName>MyAssemblyName</AssemblyName>
<EnableUnmanagedDebugging>false</EnableUnmanagedDebugging>
<OutputPath>bin\</OutputPath>
<Name>AsPartitionProcessing.AdventureWorks</Name>
<DeploymentServerName>localhost</DeploymentServerName>
<DeploymentServerEdition>Developer</DeploymentServerEdition>
<DeploymentServerVersion>Version_11_0</DeploymentServerVersion>
<DeploymentServerDatabase>AW Tabular Model SQL 2014</DeploymentServerDatabase>
<DeploymentServerCubeName>Model</DeploymentServerCubeName>
<DeploymentOptionProcessing>Default</DeploymentOptionProcessing>
<DeploymentOptionTransactionalDeployment>False</DeploymentOptionTransactionalDeployment>
<DeploymentOptionDirectQueryMode>InMemory</DeploymentOptionDirectQueryMode>
<DeploymentOptionQueryImpersonation>Default</DeploymentOptionQueryImpersonation>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)' == 'Development' ">
<OutputPath>bin\</OutputPath>
<DeploymentServerName>localhost</DeploymentServerName>
<DeploymentServerEdition>Developer</DeploymentServerEdition>
<DeploymentServerVersion>Version_11_0</DeploymentServerVersion>
<DeploymentServerDatabase>AdventureWorks</DeploymentServerDatabase>
<DeploymentServerCubeName>Model</DeploymentServerCubeName>
<DeploymentOptionProcessing>Default</DeploymentOptionProcessing>
<DeploymentOptionTransactionalDeployment>False</DeploymentOptionTransactionalDeployment>
<DeploymentOptionDirectQueryMode>InMemory</DeploymentOptionDirectQueryMode>
<DeploymentOptionQueryImpersonation>Default</DeploymentOptionQueryImpersonation>
</PropertyGroup>
<ItemGroup />
<ItemGroup>
<Compile Include="Model.bim">
<SubType>Code</SubType>
</Compile>
</ItemGroup>
<Import Project="$(MSBuildExtensionsPath)\Business Intelligence Semantic Model\1.0\Microsoft.AnalysisServices.VSHostBuilder.targets" />
</Project>

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,24 @@
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<configSections>
<sectionGroup name="userSettings" type="System.Configuration.UserSettingsGroup, System, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089" >
<section name="AsPartitionProcessing.SampleClient.Settings" type="System.Configuration.ClientSettingsSection, System, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089" allowExeDefinition="MachineToLocalUser" requirePermission="false" />
</sectionGroup>
</configSections>
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5.2"/>
</startup>
<userSettings>
<AsPartitionProcessing.SampleClient.Settings>
<setting name="ConfigServer" serializeAs="String">
<value>localhost</value>
</setting>
<setting name="ConfigDatabase" serializeAs="String">
<value>AsPartitionProcessing</value>
</setting>
<setting name="ConfigDatabaseIntegratedAuth" serializeAs="String">
<value>True</value>
</setting>
</AsPartitionProcessing.SampleClient.Settings>
</userSettings>
</configuration>

View File

@ -0,0 +1,88 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{C45B329D-F606-4F89-A41A-785C247F24B2}</ProjectGuid>
<OutputType>Exe</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>AsPartitionProcessing.SampleClient</RootNamespace>
<AssemblyName>AsPartitionProcessing.SampleClient</AssemblyName>
<TargetFrameworkVersion>v4.5.2</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
<TargetFrameworkProfile />
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<PlatformTarget>AnyCPU</PlatformTarget>
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<PlatformTarget>AnyCPU</PlatformTarget>
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<Reference Include="Microsoft.AnalysisServices, Version=13.0.0.0, Culture=neutral, PublicKeyToken=89845dcd8080cc91, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\..\..\..\Program Files (x86)\Microsoft SQL Server\130\SDK\Assemblies\Microsoft.AnalysisServices.DLL</HintPath>
</Reference>
<Reference Include="Microsoft.AnalysisServices.Core, Version=13.0.0.0, Culture=neutral, PublicKeyToken=89845dcd8080cc91, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\..\..\..\Program Files (x86)\Microsoft SQL Server\130\SDK\Assemblies\Microsoft.AnalysisServices.Core.DLL</HintPath>
</Reference>
<Reference Include="Microsoft.AnalysisServices.Tabular, Version=13.0.0.0, Culture=neutral, PublicKeyToken=89845dcd8080cc91, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\..\..\..\Program Files (x86)\Microsoft SQL Server\130\SDK\Assemblies\Microsoft.AnalysisServices.Tabular.DLL</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Data" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="Settings.Designer.cs">
<AutoGen>True</AutoGen>
<DesignTimeSharedInput>True</DesignTimeSharedInput>
<DependentUpon>Settings.settings</DependentUpon>
</Compile>
</ItemGroup>
<ItemGroup>
<None Include="App.config" />
<None Include="Settings.settings">
<Generator>SettingsSingleFileGenerator</Generator>
<LastGenOutput>Settings.Designer.cs</LastGenOutput>
</None>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\AsPartitionProcessing\AsPartitionProcessing.csproj">
<Project>{fb937281-b06d-47fb-9846-e97b0287afce}</Project>
<Name>AsPartitionProcessing</Name>
</ProjectReference>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">
</Target>
<Target Name="AfterBuild">
</Target>
-->
</Project>

View File

@ -0,0 +1,234 @@
using System;
using System.Collections.Generic;
using Microsoft.AnalysisServices.Tabular;
namespace AsPartitionProcessing.SampleClient
{
class Program
{
const bool UseDatabase = false;
static void Main(string[] args)
{
try
{
List<PartitionedModelConfig> modelsConfig;
if (!UseDatabase)
{
modelsConfig = InitializeAdventureWorksInline();
}
else
{
modelsConfig = InitializeFromDatabase();
}
foreach (PartitionedModelConfig modelConfig in modelsConfig)
{
if (!modelConfig.IntegratedAuth) //For Azure AS
{
Console.WriteLine();
Console.Write("User name for AS server: ");
modelConfig.UserName = Console.ReadLine();
Console.Write("Password for AS server: ");
modelConfig.Password = ReadPassword();
}
//Most important method:
PartitionProcessor.PerformProcessing(modelConfig, LogMessage);
}
}
catch (Exception exc)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine();
Console.WriteLine(exc.Message, null);
}
Console.WriteLine("Press any key to exit.");
Console.ReadKey();
}
private static List<PartitionedModelConfig> InitializeAdventureWorksInline()
{
PartitionedModelConfig partitionedModel = new PartitionedModelConfig(
partitionedModelConfigID: 1,
analysisServicesServer: "localhost",
analysisServicesDatabase: "AdventureWorks",
initialSetUp: true,
incrementalOnline: true,
incrementalParallelTables: true,
integratedAuth: true,
userName: "",
password: "",
partitionedTables:
new List<PartitionedTableConfig>
{
new PartitionedTableConfig(
partitionedTableConfigID: 1,
maxDate: Convert.ToDateTime("2012-12-01"),
granularity: Granularity.Monthly,
numberOfPartitionsFull: 12,
numberOfPartitionsForIncrementalProcess: 3,
analysisServicesTable: "Internet Sales",
sourceTableName: "[dbo].[FactInternetSales]",
sourcePartitionColumn: "OrderDateKey"
),
new PartitionedTableConfig(
partitionedTableConfigID: 2,
maxDate: Convert.ToDateTime("2012-12-01"),
granularity: Granularity.Yearly,
numberOfPartitionsFull: 3,
numberOfPartitionsForIncrementalProcess: 1,
analysisServicesTable: "Reseller Sales",
sourceTableName: "[dbo].[FactResellerSales]",
sourcePartitionColumn: "OrderDateKey"
)
}
);
#region Not needed as sample includes pre-prepared version of AdventureWorks
////This section not to be used normally - just to get started with AdventureWorks. It removes existing partitions that come in AdventureWorks and creates a template one to align with assumptions listed at top of PartitionProcessor.cs file.
//if (partitionedModel.InitialSetUp)
//{
// Console.WriteLine("Initialize AdventureWorks template partitions? [y/n]");
// if (Console.ReadLine().ToLower() == "y")
// InitializeAdventureWorksDatabase(partitionedModel);
//}
#endregion
return new List<PartitionedModelConfig> { partitionedModel };
}
private static List<PartitionedModelConfig> InitializeFromDatabase()
{
ConfigDatabaseConnectionInfo connectionInfo = new ConfigDatabaseConnectionInfo();
connectionInfo.Server = Settings.Default.ConfigServer;
connectionInfo.Database = Settings.Default.ConfigDatabase;
connectionInfo.IntegratedAuth = Settings.Default.ConfigDatabaseIntegratedAuth;
if (!Settings.Default.ConfigDatabaseIntegratedAuth)
{
Console.Write("User name for config database: ");
connectionInfo.UserName = Console.ReadLine();
Console.Write("Password for config database: ");
connectionInfo.Password = ReadPassword();
}
return ConfigDatabaseHelper.ReadConfig(connectionInfo);
}
private static void LogMessage(string message, PartitionedModelConfig partitionedModel)
{
//Can provide custom logger here
try
{
if (UseDatabase)
ConfigDatabaseHelper.LogMessage(message, partitionedModel);
Console.WriteLine(message);
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
Console.WriteLine("Press any key to exit.");
Console.ReadKey();
Environment.Exit(0); //Avoid recursion if errored connecting to db
}
}
public static string ReadPassword()
{
string password = "";
ConsoleKeyInfo info = Console.ReadKey(true);
while (info.Key != ConsoleKey.Enter)
{
if (info.Key != ConsoleKey.Backspace)
{
Console.Write("*");
password += info.KeyChar;
}
else if (info.Key == ConsoleKey.Backspace)
{
if (!string.IsNullOrEmpty(password))
{
// remove one character from the list of password characters
password = password.Substring(0, password.Length - 1);
// get the location of the cursor
int pos = Console.CursorLeft;
// move the cursor to the left by one character
Console.SetCursorPosition(pos - 1, Console.CursorTop);
// replace it with space
Console.Write(" ");
// move the cursor to the left by one character again
Console.SetCursorPosition(pos - 1, Console.CursorTop);
}
}
info = Console.ReadKey(true);
}
// add a new line because user pressed enter at the end of password
Console.WriteLine();
return password;
}
#region Not needed as sample includes pre-prepared version of AdventureWorks
private static void InitializeAdventureWorksDatabase(PartitionedModelConfig parameters)
{
//In order to align with assumptions listed in PartitionProcessor.cs, need to:
//1. Delete existing partitions in InternetSales and ResellerSales
//2. Create template partition (again, see comments at top of PartitionProcessor.cs)
Console.WriteLine("Initializing AdventureWorks ...");
using (Server server = new Server())
{
//Connect and get main objects
string serverConnectionString;
if (parameters.IntegratedAuth)
serverConnectionString = $"Provider=MSOLAP;Data Source={parameters.AnalysisServicesServer};";
else
{
serverConnectionString = $"Provider=MSOLAP;Data Source={parameters.AnalysisServicesServer};User ID={parameters.UserName};Password={parameters.Password};Persist Security Info=True;Impersonation Level=Impersonate;";
}
server.Connect(serverConnectionString);
Database database = server.Databases.FindByName(parameters.AnalysisServicesDatabase);
if (database == null)
{
throw new Microsoft.AnalysisServices.ConnectionException($"Could not connect to database {parameters.AnalysisServicesDatabase}.");
}
InitializeAdventureWorksTable(database, "Internet Sales", "[dbo].[FactInternetSales]");
InitializeAdventureWorksTable(database, "Reseller Sales", "[dbo].[FactResellerSales]");
database.Update(Microsoft.AnalysisServices.UpdateOptions.ExpandFull);
server.Disconnect();
}
}
private static void InitializeAdventureWorksTable(Database database, string analysisServicesTableName, string sourceFactTableName)
{
Table table = database.Model.Tables.Find(analysisServicesTableName);
if (table == null)
{
throw new Microsoft.AnalysisServices.ConnectionException($"Could not connect to table {analysisServicesTableName}.");
}
table.Partitions.Clear();
Partition templatePartition = new Partition();
templatePartition.Name = analysisServicesTableName;
table.Partitions.Add(templatePartition);
templatePartition.Source = new QueryPartitionSource()
{
DataSource = database.Model.DataSources[0], //Assuming this is only data source (SqlServer localhost)
Query = $"SELECT * FROM {sourceFactTableName}"
};
}
#endregion
}
}

View File

@ -0,0 +1,36 @@
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("AsPartitionProcessing.SampleClient")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("Microsoft Corporation")]
[assembly: AssemblyProduct("AsPartitionProcessing.SampleClient")]
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation 2016")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("c45b329d-f606-4f89-a41a-785c247f24b2")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]

View File

@ -0,0 +1,62 @@
//------------------------------------------------------------------------------
// <auto-generated>
// This code was generated by a tool.
// Runtime Version:4.0.30319.42000
//
// Changes to this file may cause incorrect behavior and will be lost if
// the code is regenerated.
// </auto-generated>
//------------------------------------------------------------------------------
namespace AsPartitionProcessing.SampleClient {
[global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()]
[global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.Editors.SettingsDesigner.SettingsSingleFileGenerator", "14.0.0.0")]
internal sealed partial class Settings : global::System.Configuration.ApplicationSettingsBase {
private static Settings defaultInstance = ((Settings)(global::System.Configuration.ApplicationSettingsBase.Synchronized(new Settings())));
public static Settings Default {
get {
return defaultInstance;
}
}
[global::System.Configuration.UserScopedSettingAttribute()]
[global::System.Diagnostics.DebuggerNonUserCodeAttribute()]
[global::System.Configuration.DefaultSettingValueAttribute("localhost")]
public string ConfigServer {
get {
return ((string)(this["ConfigServer"]));
}
set {
this["ConfigServer"] = value;
}
}
[global::System.Configuration.UserScopedSettingAttribute()]
[global::System.Diagnostics.DebuggerNonUserCodeAttribute()]
[global::System.Configuration.DefaultSettingValueAttribute("AsPartitionProcessing")]
public string ConfigDatabase {
get {
return ((string)(this["ConfigDatabase"]));
}
set {
this["ConfigDatabase"] = value;
}
}
[global::System.Configuration.UserScopedSettingAttribute()]
[global::System.Diagnostics.DebuggerNonUserCodeAttribute()]
[global::System.Configuration.DefaultSettingValueAttribute("True")]
public bool ConfigDatabaseIntegratedAuth {
get {
return ((bool)(this["ConfigDatabaseIntegratedAuth"]));
}
set {
this["ConfigDatabaseIntegratedAuth"] = value;
}
}
}
}

View File

@ -0,0 +1,15 @@
<?xml version='1.0' encoding='utf-8'?>
<SettingsFile xmlns="http://schemas.microsoft.com/VisualStudio/2004/01/settings" CurrentProfile="(Default)" GeneratedClassNamespace="AsPartitionProcessing.SampleClient" GeneratedClassName="Settings">
<Profiles />
<Settings>
<Setting Name="ConfigServer" Type="System.String" Scope="User">
<Value Profile="(Default)">localhost</Value>
</Setting>
<Setting Name="ConfigDatabase" Type="System.String" Scope="User">
<Value Profile="(Default)">AsPartitionProcessing</Value>
</Setting>
<Setting Name="ConfigDatabaseIntegratedAuth" Type="System.Boolean" Scope="User">
<Value Profile="(Default)">True</Value>
</Setting>
</Settings>
</SettingsFile>

View File

@ -0,0 +1,73 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 14
VisualStudioVersion = 14.0.25420.1
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AsPartitionProcessing", "AsPartitionProcessing\AsPartitionProcessing.csproj", "{FB937281-B06D-47FB-9846-E97B0287AFCE}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AsPartitionProcessing.SampleClient", "AsPartitionProcessing.SampleClient\AsPartitionProcessing.SampleClient.csproj", "{C45B329D-F606-4F89-A41A-785C247F24B2}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{0B23151E-8A6B-44AA-B670-797D1EB8B3C0}"
ProjectSection(SolutionItems) = preProject
AdventureWorksDW.bak = AdventureWorksDW.bak
CreateDatabaseObjects.sql = CreateDatabaseObjects.sql
SampleConfiguration.sql = SampleConfiguration.sql
EndProjectSection
EndProject
Project("{6870E480-7721-4708-BFB8-9AE898AA21B3}") = "AsPartitionProcessing.AdventureWorks", "AsPartitionProcessing.AdventureWorks\AsPartitionProcessing.AdventureWorks.smproj", "{7274EFCC-E7DF-40AD-83DF-1DFD5B954E67}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Debug|x86 = Debug|x86
Development|Any CPU = Development|Any CPU
Development|x86 = Development|x86
Release|Any CPU = Release|Any CPU
Release|x86 = Release|x86
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{FB937281-B06D-47FB-9846-E97B0287AFCE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{FB937281-B06D-47FB-9846-E97B0287AFCE}.Debug|Any CPU.Build.0 = Debug|Any CPU
{FB937281-B06D-47FB-9846-E97B0287AFCE}.Debug|x86.ActiveCfg = Debug|Any CPU
{FB937281-B06D-47FB-9846-E97B0287AFCE}.Debug|x86.Build.0 = Debug|Any CPU
{FB937281-B06D-47FB-9846-E97B0287AFCE}.Development|Any CPU.ActiveCfg = Debug|Any CPU
{FB937281-B06D-47FB-9846-E97B0287AFCE}.Development|Any CPU.Build.0 = Debug|Any CPU
{FB937281-B06D-47FB-9846-E97B0287AFCE}.Development|x86.ActiveCfg = Debug|Any CPU
{FB937281-B06D-47FB-9846-E97B0287AFCE}.Development|x86.Build.0 = Debug|Any CPU
{FB937281-B06D-47FB-9846-E97B0287AFCE}.Release|Any CPU.ActiveCfg = Release|Any CPU
{FB937281-B06D-47FB-9846-E97B0287AFCE}.Release|Any CPU.Build.0 = Release|Any CPU
{FB937281-B06D-47FB-9846-E97B0287AFCE}.Release|x86.ActiveCfg = Release|Any CPU
{FB937281-B06D-47FB-9846-E97B0287AFCE}.Release|x86.Build.0 = Release|Any CPU
{C45B329D-F606-4F89-A41A-785C247F24B2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{C45B329D-F606-4F89-A41A-785C247F24B2}.Debug|Any CPU.Build.0 = Debug|Any CPU
{C45B329D-F606-4F89-A41A-785C247F24B2}.Debug|x86.ActiveCfg = Debug|Any CPU
{C45B329D-F606-4F89-A41A-785C247F24B2}.Debug|x86.Build.0 = Debug|Any CPU
{C45B329D-F606-4F89-A41A-785C247F24B2}.Development|Any CPU.ActiveCfg = Debug|Any CPU
{C45B329D-F606-4F89-A41A-785C247F24B2}.Development|Any CPU.Build.0 = Debug|Any CPU
{C45B329D-F606-4F89-A41A-785C247F24B2}.Development|x86.ActiveCfg = Debug|Any CPU
{C45B329D-F606-4F89-A41A-785C247F24B2}.Development|x86.Build.0 = Debug|Any CPU
{C45B329D-F606-4F89-A41A-785C247F24B2}.Release|Any CPU.ActiveCfg = Release|Any CPU
{C45B329D-F606-4F89-A41A-785C247F24B2}.Release|Any CPU.Build.0 = Release|Any CPU
{C45B329D-F606-4F89-A41A-785C247F24B2}.Release|x86.ActiveCfg = Release|Any CPU
{C45B329D-F606-4F89-A41A-785C247F24B2}.Release|x86.Build.0 = Release|Any CPU
{7274EFCC-E7DF-40AD-83DF-1DFD5B954E67}.Debug|Any CPU.ActiveCfg = Development|x86
{7274EFCC-E7DF-40AD-83DF-1DFD5B954E67}.Debug|Any CPU.Build.0 = Development|x86
{7274EFCC-E7DF-40AD-83DF-1DFD5B954E67}.Debug|Any CPU.Deploy.0 = Development|x86
{7274EFCC-E7DF-40AD-83DF-1DFD5B954E67}.Debug|x86.ActiveCfg = Development|x86
{7274EFCC-E7DF-40AD-83DF-1DFD5B954E67}.Debug|x86.Build.0 = Development|x86
{7274EFCC-E7DF-40AD-83DF-1DFD5B954E67}.Debug|x86.Deploy.0 = Development|x86
{7274EFCC-E7DF-40AD-83DF-1DFD5B954E67}.Development|Any CPU.ActiveCfg = Development|x86
{7274EFCC-E7DF-40AD-83DF-1DFD5B954E67}.Development|x86.ActiveCfg = Development|x86
{7274EFCC-E7DF-40AD-83DF-1DFD5B954E67}.Development|x86.Build.0 = Development|x86
{7274EFCC-E7DF-40AD-83DF-1DFD5B954E67}.Development|x86.Deploy.0 = Development|x86
{7274EFCC-E7DF-40AD-83DF-1DFD5B954E67}.Release|Any CPU.ActiveCfg = Development|x86
{7274EFCC-E7DF-40AD-83DF-1DFD5B954E67}.Release|Any CPU.Build.0 = Development|x86
{7274EFCC-E7DF-40AD-83DF-1DFD5B954E67}.Release|Any CPU.Deploy.0 = Development|x86
{7274EFCC-E7DF-40AD-83DF-1DFD5B954E67}.Release|x86.ActiveCfg = Development|x86
{7274EFCC-E7DF-40AD-83DF-1DFD5B954E67}.Release|x86.Build.0 = Development|x86
{7274EFCC-E7DF-40AD-83DF-1DFD5B954E67}.Release|x86.Deploy.0 = Development|x86
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
EndGlobal

View File

@ -0,0 +1,72 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{FB937281-B06D-47FB-9846-E97B0287AFCE}</ProjectGuid>
<OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>AsPartitionProcessing</RootNamespace>
<AssemblyName>AsPartitionProcessing</AssemblyName>
<TargetFrameworkVersion>v4.5.2</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<TargetFrameworkProfile />
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<DocumentationFile>bin\Release\AsPartitionProcessing.XML</DocumentationFile>
</PropertyGroup>
<ItemGroup>
<Reference Include="Microsoft.AnalysisServices, Version=13.0.0.0, Culture=neutral, PublicKeyToken=89845dcd8080cc91, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\..\..\..\Program Files (x86)\Microsoft SQL Server\130\SDK\Assemblies\Microsoft.AnalysisServices.DLL</HintPath>
</Reference>
<Reference Include="Microsoft.AnalysisServices.Core, Version=13.0.0.0, Culture=neutral, PublicKeyToken=89845dcd8080cc91, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\..\..\..\Program Files (x86)\Microsoft SQL Server\130\SDK\Assemblies\Microsoft.AnalysisServices.Core.DLL</HintPath>
</Reference>
<Reference Include="Microsoft.AnalysisServices.Tabular, Version=13.0.0.0, Culture=neutral, PublicKeyToken=89845dcd8080cc91, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\..\..\..\Program Files (x86)\Microsoft SQL Server\130\SDK\Assemblies\Microsoft.AnalysisServices.Tabular.DLL</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Data" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="ConfigDatabaseConnectionInfo.cs" />
<Compile Include="PartitionedModelConfig.cs" />
<Compile Include="ConfigDatabaseHelper.cs" />
<Compile Include="PartitionedTableConfig.cs" />
<Compile Include="PartitionProcessor.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">
</Target>
<Target Name="AfterBuild">
</Target>
-->
</Project>

View File

@ -0,0 +1,35 @@
using System;
namespace AsPartitionProcessing
{
/// <summary>
/// Information required to connect to the configuration and logging database.
/// </summary>
public class ConfigDatabaseConnectionInfo
{
/// <summary>
/// Database server name.
/// </summary>
public string Server { get; set; }
/// <summary>
/// Name of the database.
/// </summary>
public string Database { get; set; }
/// <summary>
/// User name used for connection.
/// </summary>
public string UserName { get; set; }
/// <summary>
/// Password used for connection.
/// </summary>
public string Password { get; set; }
/// <summary>
/// Whether connection to be made using integrated authentication or SQL authentication.
/// </summary>
public bool IntegratedAuth { get; set; }
}
}

View File

@ -0,0 +1,173 @@
using System;
using System.Data;
using System.Data.SqlClient;
using System.Collections.Generic;
namespace AsPartitionProcessing
{
/// <summary>
/// Class containing helper methods for reading and writing to the configuration and logging database.
/// </summary>
public static class ConfigDatabaseHelper
{
/// <summary>
/// Read configuration information from the database.
/// </summary>
/// <param name="connectionInfo">Information required to connect to the configuration and logging database.</param>
/// <returns>Collection of partitioned models with configuration information.</returns>
public static List<PartitionedModelConfig> ReadConfig(ConfigDatabaseConnectionInfo connectionInfo)
{
using (SqlConnection connection = new SqlConnection(GetConnectionString(connectionInfo)))
{
connection.Open();
using (SqlCommand command = new SqlCommand())
{
command.Connection = connection;
command.CommandType = CommandType.Text;
command.CommandText = @"
SELECT [PartitionedModelConfigID]
,[AnalysisServicesServer]
,[AnalysisServicesDatabase]
,[InitialSetUp]
,[IncrementalOnline]
,[IncrementalParallelTables]
,[IntegratedAuth]
,[PartitionedTableConfigID]
,[MaxDate]
,[Granularity]
,[NumberOfPartitionsFull]
,[NumberOfPartitionsForIncrementalProcess]
,[AnalysisServicesTable]
,[SourceTableName]
,[SourcePartitionColumn]
FROM [dbo].[vPartitionedTableConfig]
ORDER BY [PartitionedModelConfigID], [PartitionedTableConfigID];";
List<PartitionedModelConfig> models = new List<PartitionedModelConfig>();
PartitionedModelConfig modelConfig = null;
int currentPartitionedModelConfigID = -1;
SqlDataReader reader = command.ExecuteReader();
while (reader.Read())
{
if (modelConfig == null || currentPartitionedModelConfigID != Convert.ToInt32(reader["PartitionedModelConfigID"]))
{
modelConfig = new PartitionedModelConfig();
modelConfig.PartitionedTables = new List<PartitionedTableConfig>();
models.Add(modelConfig);
modelConfig.PartitionedModelConfigID = Convert.ToInt32(reader["PartitionedModelConfigID"]);
modelConfig.AnalysisServicesServer = Convert.ToString(reader["AnalysisServicesServer"]);
modelConfig.AnalysisServicesDatabase = Convert.ToString(reader["AnalysisServicesDatabase"]);
modelConfig.InitialSetUp = Convert.ToBoolean(reader["InitialSetUp"]);
modelConfig.IncrementalOnline = Convert.ToBoolean(reader["IncrementalOnline"]);
modelConfig.IncrementalParallelTables = Convert.ToBoolean(reader["IncrementalParallelTables"]);
modelConfig.IntegratedAuth = Convert.ToBoolean(reader["IntegratedAuth"]);
modelConfig.ConfigDatabaseConnectionInfo = connectionInfo;
currentPartitionedModelConfigID = modelConfig.PartitionedModelConfigID;
}
modelConfig.PartitionedTables.Add(
new PartitionedTableConfig(
Convert.ToInt32(reader["PartitionedTableConfigID"]),
Convert.ToDateTime(reader["MaxDate"]),
(Granularity)Convert.ToInt32(reader["Granularity"]),
Convert.ToInt32(reader["NumberOfPartitionsFull"]),
Convert.ToInt32(reader["NumberOfPartitionsForIncrementalProcess"]),
Convert.ToString(reader["AnalysisServicesTable"]),
Convert.ToString(reader["SourceTableName"]),
Convert.ToString(reader["SourcePartitionColumn"])
)
);
}
return models;
}
}
}
/// <summary>
/// Delete all existing logs from the database. Useful in demo scenarios to initialize the database.
/// </summary>
/// <param name="connectionInfo">Information required to connect to the configuration and logging database.</param>
public static void ClearLogTable(ConfigDatabaseConnectionInfo connectionInfo)
{
using (var connection = new SqlConnection(GetConnectionString(connectionInfo)))
{
connection.Open();
using (var command = new SqlCommand())
{
command.Connection = connection;
command.CommandType = CommandType.Text;
command.CommandText = "DELETE FROM [dbo].[PartitionedModelLog];";
command.ExecuteNonQuery();
}
}
}
/// <summary>
/// Log a message to the databsae.
/// </summary>
/// <param name="message">Message to be logged.</param>
/// <param name="partitionedModel">Partitioned model with configuration information.</param>
public static void LogMessage(string message, PartitionedModelConfig partitionedModel)
{
using (var connection = new SqlConnection(GetConnectionString(partitionedModel.ConfigDatabaseConnectionInfo)))
{
connection.Open();
using (var command = new SqlCommand())
{
command.Connection = connection;
command.CommandType = CommandType.Text;
command.CommandText = @"
INSERT INTO [dbo].[PartitionedModelLog]
([PartitionedModelConfigID]
,[ExecutionID]
,[LogDateTime]
,[Message])
VALUES
(@PartitionedModelConfigID
,@ExecutionID
,@LogDateTime
,@Message);";
SqlParameter parameter;
parameter = new SqlParameter("@PartitionedModelConfigID", SqlDbType.Int);
parameter.Value = partitionedModel.PartitionedModelConfigID;
command.Parameters.Add(parameter);
parameter = new SqlParameter("@ExecutionID", SqlDbType.Char, 36);
parameter.Value = partitionedModel.ExecutionID;
command.Parameters.Add(parameter);
parameter = new SqlParameter("@LogDateTime", SqlDbType.DateTime);
parameter.Value = DateTime.Now;
command.Parameters.Add(parameter);
parameter = new SqlParameter("@Message", SqlDbType.VarChar, 4000);
parameter.Value = message;
command.Parameters.Add(parameter);
command.ExecuteNonQuery();
}
}
}
private static string GetConnectionString(ConfigDatabaseConnectionInfo connectionInfo)
{
string connectionString;
if (connectionInfo.IntegratedAuth)
{
connectionString = $"Server={connectionInfo.Server};Database={connectionInfo.Database};Integrated Security=SSPI;";
}
else
{
connectionString = $"Server={connectionInfo.Server};Database={connectionInfo.Database};User ID={connectionInfo.UserName};Password={connectionInfo.Password};";
}
return connectionString;
}
}
}

View File

@ -0,0 +1,14 @@
using System;
namespace AsPartitionProcessing
{
public class LogMessageEventArgs : EventArgs
{
public string Message { get; set; }
public LogMessageEventArgs(string message)
{
Message = message;
}
}
}

View File

@ -0,0 +1,343 @@
using System;
using System.Collections.Generic;
using Microsoft.AnalysisServices.Tabular;
//-----------
//ASSUMPTIONS
//Rolling window. Removes oldest partition on increment
//Depends on date keys in source to be integers formatted as yyyymmdd
//Source queries take the form "SELECT * FROM <SourceTable> WHERE FLOOR(<SourceColumn> / 100) = <yyyymm>" (monthly)
//Template partition exists with same name as table
//Non-template partitions have name of the format yyyy (yearly), yyyymm (monthly), yyyymmdd (daily)
//-----------
namespace AsPartitionProcessing
{
/// <summary>
/// Delegate to allow client to pass in a custom logging method
/// </summary>
/// <param name="message">The message to be logged</param>
/// <param name="partitionedModel">Configuration info for the partitioned model</param>
public delegate void LogMessageDelegate(string message, PartitionedModelConfig partitionedModel);
/// <summary>
/// Processor of partitions in AS tabular models
/// </summary>
public static class PartitionProcessor
{
private static PartitionedModelConfig _partitionedModel;
private static LogMessageDelegate _messageLogger;
/// <summary>
/// Partitions tables in a tabular model based on configuration
/// </summary>
/// <param name="partitionedModel">Configuration info for the partitioned model</param>
/// <param name="messageLogger">Pointer to logging method</param>
public static void PerformProcessing(PartitionedModelConfig partitionedModel, LogMessageDelegate messageLogger)
{
_partitionedModel = partitionedModel;
_messageLogger = messageLogger;
Server server = new Server();
try
{
Database database;
Connect(server, out database);
Console.ForegroundColor = ConsoleColor.White;
LogMessage($"Start: {DateTime.Now.ToString("hh:mm:ss tt")}", false);
LogMessage($"Server: {_partitionedModel.AnalysisServicesServer}", false);
LogMessage($"Database: {_partitionedModel.AnalysisServicesDatabase}", false);
foreach (PartitionedTableConfig partitionedTable in _partitionedModel.PartitionedTables)
{
Console.ForegroundColor = ConsoleColor.Yellow;
Table table = database.Model.Tables.Find(partitionedTable.AnalysisServicesTable);
if (table == null)
{
throw new Microsoft.AnalysisServices.ConnectionException($"Could not connect to table {partitionedTable.AnalysisServicesTable}.");
}
Partition templatePartition = table.Partitions.Find(partitionedTable.AnalysisServicesTable);
if (templatePartition == null)
{
throw new Microsoft.AnalysisServices.ConnectionException($"Table {partitionedTable.AnalysisServicesTable} does not contain a partition with the same name to act as the template partition.");
}
LogMessage("", false);
LogMessage($"Rolling-window partitioning for table {partitionedTable.AnalysisServicesTable}", false);
LogMessage(new String('-', partitionedTable.AnalysisServicesTable.Length + 38), false);
//Figure out what processing needs to be done
List<string> partitionKeysCurrent = GetPartitionKeysTable(table, partitionedTable.Granularity);
List<string> partitionKeysNew = GetPartitionKeys(false, partitionedTable, partitionedTable.Granularity);
List<string> partitionKeysForProcessing = GetPartitionKeys(true, partitionedTable, partitionedTable.Granularity);
DisplayPartitionRange(partitionKeysCurrent, true, partitionedTable.Granularity);
DisplayPartitionRange(partitionKeysNew, false, partitionedTable.Granularity);
LogMessage("", false);
LogMessage("=>Actions & progress:", false);
//Check for old partitions that need to be removed
foreach (string partitionKey in partitionKeysCurrent)
{
if (Convert.ToInt32(partitionKey) < Convert.ToInt32(partitionKeysNew[0]))
{
LogMessage($"Remove old partition {DateFormatPartitionKey(partitionKey, partitionedTable.Granularity)}", true);
table.Partitions.Remove(partitionKey);
}
}
//Process partitions
string selectQueryTemplate;
switch (partitionedTable.Granularity)
{
case Granularity.Daily:
selectQueryTemplate = "SELECT * FROM {0} WHERE {1} = {2} ORDER BY {1}";
break;
case Granularity.Monthly:
selectQueryTemplate = "SELECT * FROM {0} WHERE FLOOR({1} / 100) = {2} ORDER BY {1}";
break;
default: //Granularity.Yearly:
selectQueryTemplate = "SELECT * FROM {0} WHERE FLOOR({1} / 10000) = {2} ORDER BY {1}";
break;
}
foreach (string partitionKey in partitionKeysForProcessing)
{
Partition partitionToProcess = table.Partitions.Find(partitionKey);
if (partitionToProcess == null)
{
//Doesn't exist so create it
partitionToProcess = new Partition();
templatePartition.CopyTo(partitionToProcess);
partitionToProcess.Name = partitionKey;
((QueryPartitionSource)partitionToProcess.Source).Query = String.Format(selectQueryTemplate, partitionedTable.SourceTableName, partitionedTable.SourcePartitionColumn, partitionKey);
table.Partitions.Add(partitionToProcess);
LogMessage($"Create new partition {DateFormatPartitionKey(partitionKey, partitionedTable.Granularity)}", true);
if (!_partitionedModel.InitialSetUp)
{
IncrementalProcessPartition(partitionKey, partitionToProcess, partitionedTable.Granularity);
}
}
else if (!_partitionedModel.InitialSetUp)
{
//Existing partition for processing
IncrementalProcessPartition(partitionKey, partitionToProcess, partitionedTable.Granularity);
}
if (_partitionedModel.InitialSetUp)
{
if (partitionToProcess.State != ObjectState.Ready)
{
//Process new partitions sequentially during initial setup so don't run out of memory
LogMessage($"Sequentially process {DateFormatPartitionKey(partitionKey, partitionedTable.Granularity)} /DataOnly", true);
partitionToProcess.RequestRefresh(RefreshType.DataOnly);
database.Model.SaveChanges();
}
else
{
//Partition already exists during initial setup (and is fully processed), so ignore it
LogMessage($"Partition {DateFormatPartitionKey(partitionKey, partitionedTable.Granularity)} already exists and is processed", true);
}
}
}
//Ensure template partition doesn't contain any data
if (_partitionedModel.InitialSetUp)
{
((QueryPartitionSource)templatePartition.Source).Query = String.Format("SELECT * FROM {0} WHERE 0=1", partitionedTable.SourceTableName);
templatePartition.RequestRefresh(RefreshType.DataOnly);
}
//If processing tables sequentially (but all partitions being done in parallel), then save changes now
if (!_partitionedModel.IncrementalParallelTables)
{
LogMessage($"Save changes for table {partitionedTable.AnalysisServicesTable} ...", true);
database.Model.SaveChanges();
}
}
//Commit the data changes, and bring model back online if necessary
LogMessage("", false);
LogMessage("Final operations", false);
LogMessage(new String('-', 16), false);
if (_partitionedModel.IncrementalParallelTables)
{
LogMessage("Save changes ...", true);
database.Model.SaveChanges();
}
if (_partitionedModel.InitialSetUp || (!_partitionedModel.InitialSetUp && !_partitionedModel.IncrementalOnline))
{
LogMessage("Recalc model to bring back online ...", true);
database.Model.RequestRefresh(RefreshType.Calculate);
database.Model.SaveChanges();
}
Console.ForegroundColor = ConsoleColor.White;
LogMessage("", false);
LogMessage("Finish: " + DateTime.Now.ToString("hh:mm:ss tt"), false);
}
catch (Exception exc)
{
Console.ForegroundColor = ConsoleColor.Red;
LogMessage("", false);
LogMessage($"Exception occurred: {DateTime.Now.ToString("hh:mm:ss tt")}", false);
LogMessage($"Exception message: {exc.Message}", false);
LogMessage($"Inner exception message: {exc.InnerException.Message}", false);
}
finally
{
try
{
_partitionedModel = null;
_messageLogger = null;
if (server != null) server.Disconnect();
}
catch { }
}
}
private static void IncrementalProcessPartition(string partitionKey, Partition partitionToProcess, Granularity granularity)
{
if (_partitionedModel.IncrementalOnline)
{
LogMessage($"Parallel process partition {DateFormatPartitionKey(partitionKey, granularity)} /Full", true);
partitionToProcess.RequestRefresh(RefreshType.Full);
}
else
{
LogMessage($"Parallel process partition {DateFormatPartitionKey(partitionKey, granularity)} /DataOnly", true);
partitionToProcess.RequestRefresh(RefreshType.DataOnly);
}
}
private static void LogMessage(string message, bool indented)
{
_messageLogger($"{(indented ? new String(' ', 3) : "")}{message}", _partitionedModel);
}
private static string DateFormatPartitionKey(string partitionKey, Granularity granularity)
{
string returnVal = partitionKey.Substring(0, 4);
try
{
if (granularity == Granularity.Monthly || granularity == Granularity.Daily)
{
returnVal += "-" + partitionKey.Substring(4, 2);
}
if (granularity == Granularity.Daily)
{
returnVal += "-" + partitionKey.Substring(6, 2);
}
}
catch (ArgumentOutOfRangeException exc)
{
throw new InvalidOperationException($"Failed to derive date from partition key. Check the key {partitionKey} matches {Convert.ToString(granularity)} granularity.");
}
return returnVal;
}
private static void DisplayPartitionRange(List<string> partitionKeys, bool current, Granularity granularity)
{
LogMessage("", false);
if (partitionKeys.Count > 0)
{
LogMessage($"=>{(current ? "Current" : "New")} partition range ({Convert.ToString(granularity)}):", false);
LogMessage($"MIN partition: {DateFormatPartitionKey(partitionKeys[0], granularity)}", true);
LogMessage($"MAX partition: {DateFormatPartitionKey(partitionKeys[partitionKeys.Count - 1], granularity)}", true);
LogMessage($"Partition count: {partitionKeys.Count}", true);
}
else
{
LogMessage("=>Table not yet partitioned", false);
}
}
private static void Connect(Server server, out Database database)
{
//Connect and get main objects
string serverConnectionString;
if (_partitionedModel.IntegratedAuth)
serverConnectionString = $"Provider=MSOLAP;Data Source={_partitionedModel.AnalysisServicesServer};";
else
{
serverConnectionString = $"Provider=MSOLAP;Data Source={_partitionedModel.AnalysisServicesServer};User ID={_partitionedModel.UserName};Password={_partitionedModel.Password};Persist Security Info=True;Impersonation Level=Impersonate;";
}
server.Connect(serverConnectionString);
database = server.Databases.FindByName(_partitionedModel.AnalysisServicesDatabase);
if (database == null)
{
throw new Microsoft.AnalysisServices.ConnectionException($"Could not connect to database {_partitionedModel.AnalysisServicesDatabase}.");
}
}
private static List<string> GetPartitionKeys(bool forProcessing, PartitionedTableConfig partitionedTable, Granularity granularity)
{
//forProcessing: false to return complete target set of partitions, true to return partitons to be processed (may be less if incremental mode).
List<string> partitionKeys = new List<string>();
int numberOfPartitions = (forProcessing && !_partitionedModel.InitialSetUp ? partitionedTable.NumberOfPartitionsForIncrementalProcess : partitionedTable.NumberOfPartitionsFull);
for (int i = numberOfPartitions - 1; i >= 0; i--)
{
DateTime periodToAdd;
switch (granularity)
{
case Granularity.Daily:
periodToAdd = partitionedTable.MaxDate.AddDays(-i);
partitionKeys.Add(Convert.ToString((periodToAdd.Year * 100 + periodToAdd.Month) * 100 + periodToAdd.Day));
break;
case Granularity.Monthly:
periodToAdd = partitionedTable.MaxDate.AddMonths(-i);
partitionKeys.Add(Convert.ToString(periodToAdd.Year * 100 + periodToAdd.Month));
break;
default: //Granularity.Yearly:
periodToAdd = partitionedTable.MaxDate.AddYears(-i);
partitionKeys.Add(Convert.ToString(periodToAdd.Year));
break;
}
}
partitionKeys.Sort();
return partitionKeys;
}
private static List<string> GetPartitionKeysTable(Table table, Granularity granularity)
{
List<string> partitionKeysExisting = new List<string>();
foreach (Partition partition in table.Partitions)
{
//Ignore partitions that don't follow the convention yyyy, yyyymm or yyyymmdd
int partitionKey;
if (
( ( partition.Name.Length == 4 && granularity == Granularity.Yearly ) ||
( partition.Name.Length == 6 && granularity == Granularity.Monthly ) ||
( partition.Name.Length == 8 && granularity == Granularity.Daily )
) && int.TryParse(partition.Name, out partitionKey)
)
{
partitionKeysExisting.Add(Convert.ToString(partitionKey));
}
}
partitionKeysExisting.Sort();
return partitionKeysExisting;
}
}
}

View File

@ -0,0 +1,118 @@
using System;
using System.Collections.Generic;
namespace AsPartitionProcessing
{
/// <summary>
/// Configuration information for a partitioned AS tabular model.
/// </summary>
public class PartitionedModelConfig
{
/// <summary>
/// ID of the PartitionedModelConfig table.
/// </summary>
public int PartitionedModelConfigID { get; set; }
/// <summary>
/// Name of the Analysis Services instance. Can be SSAS or an Azure AS URL.
/// </summary>
public string AnalysisServicesServer { get; set; }
/// <summary>
/// Name of the Analysis Services database.
/// </summary>
public string AnalysisServicesDatabase { get; set; }
/// <summary>
/// True for initial set up to create partitions and process them sequentially. False for incremental processing.
/// </summary>
public bool InitialSetUp { get; set; }
/// <summary>
/// When initialSetUp=false, determines if processing is performed as an online operation, which may require more memory, but users can still query the model.
/// </summary>
public bool IncrementalOnline { get; set; }
/// <summary>
/// When initialSetUp=false, determines if separate tables are processed in parallel. Partitions within a table are always processed in parallel.
/// </summary>
public bool IncrementalParallelTables { get; set; }
/// <summary>
/// Should always set to true for SSAS implementations that will run under the current process account. For Azure AS, normally set to false.
/// </summary>
public bool IntegratedAuth { get; set; }
/// <summary>
/// Only applies when integratedAuth=false. Used for Azure AD UPNs to connect to Azure AS.
/// </summary>
public string UserName { get; set; }
/// <summary>
/// Only applies when integratedAuth=false. Used for Azure AD UPNs to connect to Azure AS.
/// </summary>
public string Password { get; set; }
/// <summary>
/// Collection of partitioned tables containing configuration information.
/// </summary>
public List<PartitionedTableConfig> PartitionedTables { get; set; }
/// <summary>
/// Connection information to connect to the configuration and logging database.
/// </summary>
public ConfigDatabaseConnectionInfo ConfigDatabaseConnectionInfo { get; set; }
/// <summary>
/// GUID generated to the execution run.
/// </summary>
public string ExecutionID { get; set; }
/// <summary>
/// Parameters normally from configuration database to determine partitioning ranges and design.
/// </summary>
/// <param name="partitionedModelConfigID">ID of the PartitionedModelConfig table.</param>
/// <param name="analysisServicesServer">Name of the Analysis Services instance. Can be SSAS or an Azure AS URL.</param>
/// <param name="analysisServicesDatabase">Name of the Analysis Services database.</param>
/// <param name="initialSetUp">True for initial set up to create partitions and process them sequentially. False for incremental processing.</param>
/// <param name="incrementalOnline">When initialSetUp=false, determines if processing is performed as an online operation, which may require more memory, but users can still query the model.</param>
/// <param name="incrementalParallelTables">When initialSetUp=false, determines if separate tables are processed in parallel. Partitions within a table are always processed in parallel.</param>
/// <param name="integratedAuth">Should always set to true for SSAS implementations that will run under the current process account. For Azure AS, normally set to false.</param>
/// <param name="userName">Only applies when integratedAuth=false. Used for Azure AD UPNs to connect to Azure AS.</param>
/// <param name="password">Only applies when integratedAuth=false. Used for Azure AD UPNs to connect to Azure AS.</param>
/// <param name="partitionedTables">Collection of partitioned tables containing configuration information.</param>
public PartitionedModelConfig(
int partitionedModelConfigID,
string analysisServicesServer,
string analysisServicesDatabase,
bool initialSetUp,
bool incrementalOnline,
bool incrementalParallelTables,
bool integratedAuth,
string userName,
string password,
List<PartitionedTableConfig> partitionedTables
)
{
PartitionedModelConfigID = partitionedModelConfigID;
AnalysisServicesServer = analysisServicesServer;
AnalysisServicesDatabase = analysisServicesDatabase;
InitialSetUp = initialSetUp;
IncrementalOnline = incrementalOnline;
IncrementalParallelTables = incrementalParallelTables;
IntegratedAuth = integratedAuth;
UserName = userName;
Password = password;
PartitionedTables = partitionedTables;
ExecutionID = Guid.NewGuid().ToString();
}
/// <summary>
/// Default constructor.
/// </summary>
public PartitionedModelConfig()
{
ExecutionID = Guid.NewGuid().ToString();
}
}
}

View File

@ -0,0 +1,93 @@
using System;
namespace AsPartitionProcessing
{
/// <summary>
/// Configuration information for a partitioned table within an AS tabular model.
/// </summary>
public class PartitionedTableConfig
{
/// <summary>
/// ID of the PartitionedTableConfig table.
/// </summary>
public int PartitionedTableConfigID { get; set; }
/// <summary>
/// The maximum date that needs to be accounted for in the partitioned table. Represents the upper boundary of the rolling window.
/// </summary>
public DateTime MaxDate { get; set; }
/// <summary>
/// Partition granularity, which can be Yearly, Monthly or Daily.
/// </summary>
public Granularity Granularity { get; set; }
/// <summary>
/// Count of all partitions in the rolling window. For example, a rolling window of 10 years partitioned by month would result in 120 partitions.
/// </summary>
public int NumberOfPartitionsFull { get; set; }
/// <summary>
/// Count of “hot partitions” where the data can change. For example, it may be necessary to refresh the most recent 3 months of data every day. This only applies to the most recent partitions.
/// </summary>
public int NumberOfPartitionsForIncrementalProcess { get; set; }
/// <summary>
/// Name of the partitioned table in the tabular model.
/// </summary>
public string AnalysisServicesTable { get; set; }
/// <summary>
/// Name of the source table in the relational database.
/// </summary>
public string SourceTableName { get; set; }
/// <summary>
/// Name of the source column from the table in the relational database.
/// </summary>
public string SourcePartitionColumn { get; set; }
/// <summary>
/// Initialize configuration info for partitioned table. Normally populated from a configuration database.
/// </summary>
/// <param name="model">Parent model.</param>
/// <param name="partitionedTableConfigID">ID of the PartitionedTableConfig table.</param>
/// <param name="maxDate">The maximum date that needs to be accounted for in the partitioned table. Represents the upper boundary of the rolling window.</param>
/// <param name="granularity">Partition granularity, which can be Yearly, Monthly or Daily.</param>
/// <param name="numberOfPartitionsFull">Count of all partitions in the rolling window. For example, a rolling window of 10 years partitioned by month would result in 120 partitions.</param>
/// <param name="numberOfPartitionsForIncrementalProcess">Count of “hot partitions” where the data can change. For example, it may be necessary to refresh the most recent 3 months of data every day. This only applies to the most recent partitions.</param>
/// <param name="analysisServicesTable">Name of the partitioned table in the tabular model.</param>
/// <param name="sourceTableName">Name of the source table in the relational database.</param>
/// <param name="sourcePartitionColumn">Name of the source column from the table in the relational database.</param>
public PartitionedTableConfig(
int partitionedTableConfigID,
DateTime maxDate,
Granularity granularity,
int numberOfPartitionsFull,
int numberOfPartitionsForIncrementalProcess,
string analysisServicesTable,
string sourceTableName,
string sourcePartitionColumn
)
{
PartitionedTableConfigID = partitionedTableConfigID;
MaxDate = maxDate;
Granularity = granularity;
NumberOfPartitionsFull = numberOfPartitionsFull;
NumberOfPartitionsForIncrementalProcess = numberOfPartitionsForIncrementalProcess;
AnalysisServicesTable = analysisServicesTable;
SourceTableName = sourceTableName;
SourcePartitionColumn = sourcePartitionColumn;
}
}
/// <summary>
/// Enumeration of supported partition granularities.
/// </summary>
public enum Granularity
{
Daily = 0,
Monthly = 1,
Yearly = 2
}
}

View File

@ -0,0 +1,36 @@
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("AsPartitionProcessing")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("Microsoft Corporation")]
[assembly: AssemblyProduct("AsPartitionProcessing")]
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation 2016")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("fb937281-b06d-47fb-9846-e97b0287afce")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]

View File

@ -0,0 +1,87 @@
CREATE DATABASE [AsPartitionProcessing]
GO
USE [AsPartitionProcessing]
GO
CREATE TABLE [dbo].[PartitionedModelConfig](
[PartitionedModelConfigID] [int] NOT NULL,
[AnalysisServicesServer] [varchar](255) NOT NULL,
[AnalysisServicesDatabase] [varchar](255) NOT NULL,
[InitialSetUp] [bit] NOT NULL,
[IncrementalOnline] [bit] NOT NULL,
[IncrementalParallelTables] [bit] NOT NULL,
[IntegratedAuth] [bit] NOT NULL,
CONSTRAINT [PK_PartitionedDatabaseConfig] PRIMARY KEY CLUSTERED
(
[PartitionedModelConfigID] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
) ON [PRIMARY]
GO
CREATE TABLE [dbo].[PartitionedTableConfig](
[PartitionedTableConfigID] [int] NOT NULL,
[PartitionedModelConfigID] [int] NOT NULL,
[MaxDate] [date] NOT NULL,
[Granularity] [tinyint] NOT NULL,
[NumberOfPartitionsFull] [int] NOT NULL,
[NumberOfPartitionsForIncrementalProcess] [int] NOT NULL,
[AnalysisServicesTable] [varchar](255) NOT NULL,
[SourceTableName] [varchar](255) NOT NULL,
[SourcePartitionColumn] [varchar](255) NOT NULL,
CONSTRAINT [PK_PartitionedTablesConfig] PRIMARY KEY CLUSTERED
(
[PartitionedTableConfigID] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
) ON [PRIMARY]
GO
CREATE TABLE [dbo].[PartitionedModelLog](
[PartitionedModelLogID] [int] IDENTITY(1,1) NOT NULL,
[PartitionedModelConfigID] [int] NOT NULL,
[ExecutionID] [char](36) NOT NULL,
[LogDateTime] [datetime] NOT NULL,
[Message] [varchar](8000) NOT NULL,
CONSTRAINT [PK_PartitionedModelLog] PRIMARY KEY CLUSTERED
(
[PartitionedModelLogID] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
) ON [PRIMARY]
GO
ALTER TABLE [dbo].[PartitionedTableConfig] WITH CHECK ADD CONSTRAINT [FK_PartitionedTableConfig_PartitionedDatabaseConfig] FOREIGN KEY([PartitionedModelConfigID])
REFERENCES [dbo].[PartitionedModelConfig] ([PartitionedModelConfigID])
GO
ALTER TABLE [dbo].[PartitionedTableConfig] CHECK CONSTRAINT [FK_PartitionedTableConfig_PartitionedDatabaseConfig]
GO
ALTER TABLE [dbo].[PartitionedModelLog] WITH CHECK ADD CONSTRAINT [FK_PartitionedModelLog_PartitionedModelConfig] FOREIGN KEY([PartitionedModelConfigID])
REFERENCES [dbo].[PartitionedModelConfig] ([PartitionedModelConfigID])
GO
ALTER TABLE [dbo].[PartitionedModelLog] CHECK CONSTRAINT [FK_PartitionedModelLog_PartitionedModelConfig]
GO
CREATE VIEW [dbo].[vPartitionedTableConfig]
AS
SELECT m.[PartitionedModelConfigID]
,m.[AnalysisServicesServer]
,m.[AnalysisServicesDatabase]
,m.[InitialSetUp]
,m.[IncrementalOnline]
,m.[IncrementalParallelTables]
,m.[IntegratedAuth]
,t.[PartitionedTableConfigID]
,t.[MaxDate]
,t.[Granularity]
,t.[NumberOfPartitionsFull]
,t.[NumberOfPartitionsForIncrementalProcess]
,t.[AnalysisServicesTable]
,t.[SourceTableName]
,t.[SourcePartitionColumn]
FROM [dbo].[PartitionedTableConfig] t
INNER JOIN [dbo].[PartitionedModelConfig] m ON t.[PartitionedModelConfigID] = m.[PartitionedModelConfigID]
GO

View File

@ -0,0 +1,34 @@
INSERT INTO [dbo].[PartitionedModelConfig]
VALUES(
1 --[PartitionedModelConfigID]
,'localhost' --[AnalysisServicesServer]
,'AdventureWorks' --[AnalysisServicesDatabase]
,1 --[InitialSetUp]
,1 --[IncrementalOnline]
,1 --[IncrementalParallelTables]
,1 --[IntegratedAuth]
);
INSERT INTO [dbo].[PartitionedTableConfig]
VALUES(
1 --[PartitionedTableConfigID]
,1 --[PartitionedModelConfigID]
,'2012-12-01' --[MaxDate]
,1 --[Granularity] 1=Monthly
,12 --[NumberOfPartitionsFull]
,3 --[NumberOfPartitionsForIncrementalProcess]
,'Internet Sales' --[AnalysisServicesTable]
,'[dbo].[FactInternetSales]'--[SourceTableName]
,'OrderDateKey' --[SourcePartitionColumn]
),
(
2 --[PartitionedTableConfigID]
,1 --[PartitionedModelConfigID]
,'2012-12-01' --[MaxDate]
,2 --[Granularity] 2=Yearly
,3 --[NumberOfPartitionsFull]
,1 --[NumberOfPartitionsForIncrementalProcess]
,'Reseller Sales' --[AnalysisServicesTable]
,'[dbo].[FactResellerSales]'--[SourceTableName]
,'OrderDateKey' --[SourcePartitionColumn]
);