diff --git a/AsPartitionProcessing/AsPartitionProcessing.AdventureWorks/Model.bim b/AsPartitionProcessing/AsPartitionProcessing.AdventureWorks/Model.bim
index cf4cd59..d5e461f 100644
--- a/AsPartitionProcessing/AsPartitionProcessing.AdventureWorks/Model.bim
+++ b/AsPartitionProcessing/AsPartitionProcessing.AdventureWorks/Model.bim
@@ -1,6 +1,5 @@
{
"name": "SemanticModel",
- "id": "SemanticModel",
"compatibilityLevel": 1200,
"model": {
"culture": "en-US",
@@ -3643,11 +3642,11 @@
"targetExpression": "1.1",
"statusGraphic": "Three Symbols UnCircled Colored",
"statusExpression": [
- "if(ISBLANK('Internet Sales'[Internet Current Quarter Sales Performance]),BLANK(),\r",
- " If('Internet Sales'[Internet Current Quarter Sales Performance]<1,-1,\r",
- "\t If('Internet Sales'[Internet Current Quarter Sales Performance]<1.07,0,1)\r",
- " )\r",
- " )\r",
+ "if(ISBLANK('Internet Sales'[Internet Current Quarter Sales Performance]),BLANK(),",
+ " If('Internet Sales'[Internet Current Quarter Sales Performance]<1,-1,",
+ "\t If('Internet Sales'[Internet Current Quarter Sales Performance]<1.07,0,1)",
+ " )",
+ " )",
" "
],
"annotations": [
@@ -3744,11 +3743,11 @@
"targetExpression": "1.25",
"statusGraphic": "Three Symbols UnCircled Colored",
"statusExpression": [
- "if(ISBLANK('Internet Sales'[Internet Current Quarter Gross Profit Performance]),BLANK(),\r",
- " If('Internet Sales'[Internet Current Quarter Gross Profit Performance]<0.8,-1,\r",
- "\t If('Internet Sales'[Internet Current Quarter Gross Profit Performance]<1.03,0,1)\r",
- " )\r",
- " )\r",
+ "if(ISBLANK('Internet Sales'[Internet Current Quarter Gross Profit Performance]),BLANK(),",
+ " If('Internet Sales'[Internet Current Quarter Gross Profit Performance]<0.8,-1,",
+ "\t If('Internet Sales'[Internet Current Quarter Gross Profit Performance]<1.03,0,1)",
+ " )",
+ " )",
" "
],
"annotations": [
@@ -4217,11 +4216,11 @@
"targetExpression": "1.1",
"statusGraphic": "Three Symbols UnCircled Colored",
"statusExpression": [
- "if(ISBLANK('Reseller Sales'[Reseller Current Quarter Sales Performance]),BLANK(),\r",
- " If('Reseller Sales'[Reseller Current Quarter Sales Performance]<0.8,-1,\r",
- "\t If('Reseller Sales'[Reseller Current Quarter Sales Performance]<1.07,0,1)\r",
- " )\r",
- " )\r",
+ "if(ISBLANK('Reseller Sales'[Reseller Current Quarter Sales Performance]),BLANK(),",
+ " If('Reseller Sales'[Reseller Current Quarter Sales Performance]<0.8,-1,",
+ "\t If('Reseller Sales'[Reseller Current Quarter Sales Performance]<1.07,0,1)",
+ " )",
+ " )",
" "
],
"annotations": [
@@ -4263,11 +4262,11 @@
"targetExpression": "1.25",
"statusGraphic": "Three Symbols UnCircled Colored",
"statusExpression": [
- "if(ISBLANK('Reseller Sales'[Reseller Current Quarter Gross Profit Performance]),BLANK(),\r",
- " If('Reseller Sales'[Reseller Current Quarter Gross Profit Performance]<0.8,-1,\r",
- "\t If('Reseller Sales'[Reseller Current Quarter Gross Profit Performance]<1.03,0,1)\r",
- " )\r",
- " )\r",
+ "if(ISBLANK('Reseller Sales'[Reseller Current Quarter Gross Profit Performance]),BLANK(),",
+ " If('Reseller Sales'[Reseller Current Quarter Gross Profit Performance]<0.8,-1,",
+ "\t If('Reseller Sales'[Reseller Current Quarter Gross Profit Performance]<1.03,0,1)",
+ " )",
+ " )",
" "
],
"annotations": [
@@ -6005,5 +6004,6 @@
"modelPermission": "refresh"
}
]
- }
+ },
+ "id": "SemanticModel"
}
\ No newline at end of file
diff --git a/AsPartitionProcessing/AsPartitionProcessing.SampleClient/Program.cs b/AsPartitionProcessing/AsPartitionProcessing.SampleClient/Program.cs
index 55bb35d..e387788 100644
--- a/AsPartitionProcessing/AsPartitionProcessing.SampleClient/Program.cs
+++ b/AsPartitionProcessing/AsPartitionProcessing.SampleClient/Program.cs
@@ -151,9 +151,12 @@ namespace AsPartitionProcessing.SampleClient
granularity: Granularity.Monthly,
numberOfPartitionsFull: 12,
numberOfPartitionsForIncrementalProcess: 3,
+ maxDateIsNow: false,
maxDate: Convert.ToDateTime("2012-12-01"),
- sourceTableName: "[dbo].[FactInternetSales]",
- sourcePartitionColumn: "OrderDateKey"
+ integerDateKey: true,
+ templateSourceQuery: "SELECT * FROM [dbo].[FactInternetSales] " +
+ "WHERE OrderDateKey >= {0} AND OrderDateKey < {1} " +
+ "ORDER BY OrderDateKey"
)
}
),
@@ -168,9 +171,12 @@ namespace AsPartitionProcessing.SampleClient
granularity: Granularity.Yearly,
numberOfPartitionsFull: 3,
numberOfPartitionsForIncrementalProcess: 1,
+ maxDateIsNow: false,
maxDate: Convert.ToDateTime("2012-12-01"),
- sourceTableName: "[dbo].[FactResellerSales]",
- sourcePartitionColumn: "OrderDateKey"
+ integerDateKey: true,
+ templateSourceQuery: "SELECT * FROM [dbo].[FactResellerSales] " +
+ "WHERE OrderDateKey >= {0} AND OrderDateKey < {1} " +
+ "ORDER BY OrderDateKey"
)
}
)
diff --git a/AsPartitionProcessing/AsPartitionProcessing/ConfigDatabaseHelper.cs b/AsPartitionProcessing/AsPartitionProcessing/ConfigDatabaseHelper.cs
index 7f0f9aa..995d71a 100644
--- a/AsPartitionProcessing/AsPartitionProcessing/ConfigDatabaseHelper.cs
+++ b/AsPartitionProcessing/AsPartitionProcessing/ConfigDatabaseHelper.cs
@@ -40,9 +40,10 @@ namespace AsPartitionProcessing
,[Granularity]
,[NumberOfPartitionsFull]
,[NumberOfPartitionsForIncrementalProcess]
+ ,[MaxDateIsNow]
,[MaxDate]
- ,[SourceTableName]
- ,[SourcePartitionColumn]
+ ,[IntegerDateKey]
+ ,[TemplateSourceQuery]
FROM [dbo].[vPartitioningConfiguration]
WHERE [DoNotProcess] = 0
ORDER BY
@@ -98,9 +99,10 @@ namespace AsPartitionProcessing
(Granularity)Convert.ToInt32(reader["Granularity"]),
Convert.ToInt32(reader["NumberOfPartitionsFull"]),
Convert.ToInt32(reader["NumberOfPartitionsForIncrementalProcess"]),
- Convert.ToDateTime(reader["MaxDate"]),
- Convert.ToString(reader["SourceTableName"]),
- Convert.ToString(reader["SourcePartitionColumn"])
+ Convert.ToBoolean(reader["MaxDateIsNow"]),
+ (reader["MaxDate"] == DBNull.Value ? DateTime.MinValue : Convert.ToDateTime(reader["MaxDate"])),
+ Convert.ToBoolean(reader["IntegerDateKey"]),
+ Convert.ToString(reader["TemplateSourceQuery"])
)
);
}
diff --git a/AsPartitionProcessing/AsPartitionProcessing/PartitionProcessor.cs b/AsPartitionProcessing/AsPartitionProcessing/PartitionProcessor.cs
index 4aac5c6..b7f8571 100644
--- a/AsPartitionProcessing/AsPartitionProcessing/PartitionProcessor.cs
+++ b/AsPartitionProcessing/AsPartitionProcessing/PartitionProcessor.cs
@@ -164,7 +164,7 @@ namespace AsPartitionProcessing
//Ensure template partition doesn't contain any data
if (_modelConfiguration.InitialSetUp)
{
- ((QueryPartitionSource)templatePartition.Source).Query = String.Format("SELECT * FROM {0} WHERE 0=1", tableConfiguration.PartitioningConfigurations[0].SourceTableName); //assuming the same for all partitioning configurations
+ ((QueryPartitionSource)templatePartition.Source).Query = String.Format(tableConfiguration.PartitioningConfigurations[0].TemplateSourceQuery, GetDateKey("19010102", tableConfiguration.PartitioningConfigurations[0], false), GetDateKey("19010101", tableConfiguration.PartitioningConfigurations[0], false)); //Query generated will always return nothing
templatePartition.RequestRefresh(RefreshType.DataOnly);
}
}
@@ -436,6 +436,47 @@ namespace AsPartitionProcessing
#region Private Methods
+ private static string GetDateKey(string partitionKey, PartitioningConfiguration partitioningConfiguration, bool addPeriod)
+ {
+ DateTime dateVal = new DateTime();
+
+ switch (partitioningConfiguration.Granularity)
+ {
+ case Granularity.Daily:
+ dateVal = new DateTime(Convert.ToInt32(partitionKey.Substring(0, 4)), Convert.ToInt32(partitionKey.Substring(4, 2)), Convert.ToInt32(partitionKey.Substring(6, 2)));
+ if (addPeriod)
+ {
+ dateVal = dateVal.AddDays(1);
+ }
+ break;
+ case Granularity.Monthly:
+ dateVal = new DateTime(Convert.ToInt32(partitionKey.Substring(0, 4)), Convert.ToInt32(partitionKey.Substring(4, 2)), 1);
+ if (addPeriod)
+ {
+ dateVal = dateVal.AddMonths(1);
+ }
+ break;
+ case Granularity.Yearly:
+ dateVal = new DateTime(Convert.ToInt32(partitionKey.Substring(0, 4)), 1, 1);
+ if (addPeriod)
+ {
+ dateVal = dateVal.AddYears(1);
+ }
+ break;
+ default:
+ break;
+ }
+
+ if (partitioningConfiguration.IntegerDateKey)
+ {
+ return dateVal.ToString("yyyyMMdd");
+ }
+ else
+ {
+ return $"'{dateVal.ToString("yyyy-MM-dd")}'";
+ }
+ }
+
private static void IncrementalProcessPartition(string partitionKey, Partition partitionToProcess, Granularity granularity)
{
if (_modelConfiguration.IncrementalOnline)
@@ -579,28 +620,14 @@ namespace AsPartitionProcessing
private static Partition CreateNewPartition(Table table, Partition templatePartition, PartitioningConfiguration partitioningConfiguration, string partitionKey)
{
- string selectQueryTemplate;
- switch (partitioningConfiguration.Granularity)
- {
- //Format that might work on more data sources, but requires flag to indicate whether partitioning column is date or integer YYYYMMDD or not:
- // SELECT YEAR(CURRENT_TIMESTAMP) * 10000 + MONTH(CURRENT_TIMESTAMP) * 100 + DAY(CURRENT_TIMESTAMP)
- //ANSI standard to get month from date is EXTRACT(MONTH FROM @DateTimeVarUnclean), which doesn't work with SQL Server
+ string beginParam = GetDateKey(partitionKey, partitioningConfiguration, false);
+ string endParam = GetDateKey(partitionKey, partitioningConfiguration, true);
- case Granularity.Daily:
- selectQueryTemplate = "SELECT * FROM {0} WHERE CAST(CONVERT(varchar, {1}, 112) AS int) = {2} ORDER BY {1}";
- break;
- case Granularity.Monthly:
- selectQueryTemplate = "SELECT * FROM {0} WHERE FLOOR(CAST(CONVERT(varchar, {1}, 112) AS int) / 100) = {2} ORDER BY {1}";
- break;
- default: //Granularity.Yearly:
- selectQueryTemplate = "SELECT * FROM {0} WHERE FLOOR(CAST(CONVERT(varchar, {1}, 112) AS int) / 10000) = {2} ORDER BY {1}";
- break;
- }
Partition newPartition;
newPartition = new Partition();
templatePartition.CopyTo(newPartition);
newPartition.Name = partitionKey;
- ((QueryPartitionSource)newPartition.Source).Query = String.Format(selectQueryTemplate, partitioningConfiguration.SourceTableName, partitioningConfiguration.SourcePartitionColumn, partitionKey);
+ ((QueryPartitionSource)newPartition.Source).Query = String.Format(partitioningConfiguration.TemplateSourceQuery, beginParam, endParam);
table.Partitions.Add(newPartition);
return newPartition;
}
diff --git a/AsPartitionProcessing/AsPartitionProcessing/PartitioningConfiguration.cs b/AsPartitionProcessing/AsPartitionProcessing/PartitioningConfiguration.cs
index ff260ff..79f45ac 100644
--- a/AsPartitionProcessing/AsPartitionProcessing/PartitioningConfiguration.cs
+++ b/AsPartitionProcessing/AsPartitionProcessing/PartitioningConfiguration.cs
@@ -28,7 +28,7 @@ namespace AsPartitionProcessing
public int NumberOfPartitionsForIncrementalProcess { get; set; }
///
- /// The maximum date that needs to be accounted for in the partitioned table.
+ /// If MaxDateIsNow = false, the maximum date that needs to be accounted for in the partitioned table.
///
public DateTime MaxDate { get; set; }
@@ -43,14 +43,14 @@ namespace AsPartitionProcessing
public DateTime UpperBoundary { get; }
///
- /// Name of the source table in the relational database.
+ /// Assumes date keys are integers. If false assumes date type.
///
- public string SourceTableName { get; set; }
+ public bool IntegerDateKey { get; set; }
///
- /// Name of the source column from the table in the relational database.
+ /// Template query used for partition source queries.
///
- public string SourcePartitionColumn { get; set; }
+ public string TemplateSourceQuery { get; set; }
///
/// Initialize partitioning configuration for partitioned table. Normally populated from a configuration database.
@@ -60,48 +60,58 @@ namespace AsPartitionProcessing
/// Partition granularity, which can be Yearly, Monthly or Daily.
/// Count of all partitions in the rolling window. For example, a rolling window of 10 years partitioned by month would result in 120 partitions.
/// Count of “hot partitions” where the data can change. For example, it may be necessary to refresh the most recent 3 months of data every day. This only applies to the most recent partitions.
+ /// Assumes maximum date to be accounted for is today.
/// The maximum date that needs to be accounted for in the partitioned table. Represents the upper boundary of the rolling window.
- /// Name of the source table in the relational database.
- /// Name of the source column from the table in the relational database.
+ /// Assumes date keys are integers. If false assumes date type.
+ /// Template query used for partition source queries.
public PartitioningConfiguration(
int partitioningConfigurationID,
Granularity granularity,
int numberOfPartitionsFull,
int numberOfPartitionsForIncrementalProcess,
+ bool maxDateIsNow,
DateTime maxDate,
- string sourceTableName,
- string sourcePartitionColumn
+ bool integerDateKey,
+ string templateSourceQuery
)
{
PartitioningConfigurationID = partitioningConfigurationID;
Granularity = granularity;
NumberOfPartitionsFull = numberOfPartitionsFull;
NumberOfPartitionsForIncrementalProcess = numberOfPartitionsForIncrementalProcess;
- MaxDate = maxDate;
- SourceTableName = sourceTableName;
- SourcePartitionColumn = sourcePartitionColumn;
+ if (maxDateIsNow)
+ {
+ MaxDate = DateTime.Today;
+ }
+ else
+ {
+ MaxDate = maxDate;
+ }
+ IntegerDateKey = integerDateKey;
+ TemplateSourceQuery = templateSourceQuery;
switch (granularity)
{
case Granularity.Daily:
- LowerBoundary = maxDate.AddDays(-numberOfPartitionsFull + 1);
- UpperBoundary = maxDate;
+ LowerBoundary = MaxDate.AddDays(-numberOfPartitionsFull + 1);
+ UpperBoundary = MaxDate;
break;
case Granularity.Monthly:
- LowerBoundary = Convert.ToDateTime(maxDate.AddMonths(-numberOfPartitionsFull + 1).ToString("yyyy-MMM-01"));
- UpperBoundary = Convert.ToDateTime(maxDate.AddMonths(1).ToString("yyyy-MMM-01")).AddDays(-1);
+ LowerBoundary = Convert.ToDateTime(MaxDate.AddMonths(-numberOfPartitionsFull + 1).ToString("yyyy-MMM-01"));
+ UpperBoundary = Convert.ToDateTime(MaxDate.AddMonths(1).ToString("yyyy-MMM-01")).AddDays(-1);
break;
case Granularity.Yearly:
- LowerBoundary = Convert.ToDateTime(maxDate.AddYears(-numberOfPartitionsFull + 1).ToString("yyyy-01-01"));
- UpperBoundary = Convert.ToDateTime(maxDate.AddYears(1).ToString("yyyy-01-01")).AddDays(-1);
+ LowerBoundary = Convert.ToDateTime(MaxDate.AddYears(-numberOfPartitionsFull + 1).ToString("yyyy-01-01"));
+ UpperBoundary = Convert.ToDateTime(MaxDate.AddYears(1).ToString("yyyy-01-01")).AddDays(-1);
break;
default:
break;
}
}
+
public int CompareTo(PartitioningConfiguration other) => string.Compare(this.LowerBoundary.ToString("yyyy-MM-dd"), other.LowerBoundary.ToString("yyyy-MM-dd"));
}
diff --git a/AsPartitionProcessing/CreateDatabaseObjects.sql b/AsPartitionProcessing/CreateDatabaseObjects.sql
index 738e42a..9906cfa 100644
--- a/AsPartitionProcessing/CreateDatabaseObjects.sql
+++ b/AsPartitionProcessing/CreateDatabaseObjects.sql
@@ -25,9 +25,10 @@ CREATE TABLE [dbo].[PartitioningConfiguration](
[Granularity] [tinyint] NOT NULL,
[NumberOfPartitionsFull] [int] NOT NULL,
[NumberOfPartitionsForIncrementalProcess] [int] NOT NULL,
- [MaxDate] [date] NOT NULL,
- [SourceTableName] [varchar](255) NOT NULL,
- [SourcePartitionColumn] [varchar](255) NOT NULL,
+ [MaxDateIsNow] [bit] NOT NULL,
+ [MaxDate] [date] NULL,
+ [IntegerDateKey] [bit] NOT NULL,
+ [TemplateSourceQuery] [varchar](max) NOT NULL,
CONSTRAINT [PK_PartitioningConfiguration] PRIMARY KEY CLUSTERED
(
[PartitioningConfigurationID] ASC
@@ -119,9 +120,10 @@ SELECT m.[ModelConfigurationID]
,p.[Granularity]
,p.[NumberOfPartitionsFull]
,p.[NumberOfPartitionsForIncrementalProcess]
+ ,p.[MaxDateIsNow]
,p.[MaxDate]
- ,p.[SourceTableName]
- ,p.[SourcePartitionColumn]
+ ,p.[IntegerDateKey]
+ ,p.[TemplateSourceQuery]
FROM [dbo].[ModelConfiguration] m
INNER JOIN [dbo].[TableConfiguration] t ON m.[ModelConfigurationID] = t.[ModelConfigurationID]
LEFT JOIN [dbo].[PartitioningConfiguration] p ON t.[TableConfigurationID] = p.[TableConfigurationID]
diff --git a/AsPartitionProcessing/SampleConfiguration.sql b/AsPartitionProcessing/SampleConfiguration.sql
index af27a53..b83a1c3 100644
--- a/AsPartitionProcessing/SampleConfiguration.sql
+++ b/AsPartitionProcessing/SampleConfiguration.sql
@@ -31,9 +31,10 @@ VALUES(
,1 --[Granularity] 1=Monthly
,12 --[NumberOfPartitionsFull]
,3 --[NumberOfPartitionsForIncrementalProcess]
+ ,0 --[MaxDateIsNow]
,'2012-12-01' --[MaxDate]
- ,'[dbo].[FactInternetSales]'--[SourceTableName]
- ,'OrderDateKey' --[SourcePartitionColumn]
+ ,1 --[IntegerDateKey]
+ ,'SELECT * FROM [dbo].[FactInternetSales] WHERE OrderDateKey >= {0} AND OrderDateKey < {1} ORDER BY OrderDateKey' --[TemplateSourceQuery]
),
(
2 --[PartitioningConfigurationID]
@@ -41,7 +42,8 @@ VALUES(
,2 --[Granularity] 2=Yearly
,3 --[NumberOfPartitionsFull]
,1 --[NumberOfPartitionsForIncrementalProcess]
+ ,0 --[MaxDateIsNow]
,'2012-12-01' --[MaxDate]
- ,'[dbo].[FactResellerSales]'--[SourceTableName]
- ,'OrderDateKey' --[SourcePartitionColumn]
+ ,1 --[IntegerDateKey]
+ ,'SELECT * FROM [dbo].[FactResellerSales] WHERE OrderDateKey >= {0} AND OrderDateKey < {1} ORDER BY OrderDateKey' --[TemplateSourceQuery]
);