Generic queries for different data sources

AsPartitionProcessing/AsPartitionProcessing.AdventureWorks/Model.bim

@@ -1,6 +1,5 @@
 {
   "name": "SemanticModel",
-  "id": "SemanticModel",
   "compatibilityLevel": 1200,
   "model": {
     "culture": "en-US",
@@ -3643,11 +3642,11 @@
               "targetExpression": "1.1",
               "statusGraphic": "Three Symbols UnCircled Colored",
               "statusExpression": [
-                "if(ISBLANK('Internet Sales'[Internet Current Quarter Sales Performance]),BLANK(),\r",
+                "if(ISBLANK('Internet Sales'[Internet Current Quarter Sales Performance]),BLANK(),",
-                "                    If('Internet Sales'[Internet Current Quarter Sales Performance]<1,-1,\r",
+                "                    If('Internet Sales'[Internet Current Quarter Sales Performance]<1,-1,",
-                "\t                If('Internet Sales'[Internet Current Quarter Sales Performance]<1.07,0,1)\r",
+                "\t                If('Internet Sales'[Internet Current Quarter Sales Performance]<1.07,0,1)",
-                "                )\r",
+                "                )",
-                "            )\r",
+                "            )",
                 "            "
               ],
               "annotations": [
@@ -3744,11 +3743,11 @@
               "targetExpression": "1.25",
               "statusGraphic": "Three Symbols UnCircled Colored",
               "statusExpression": [
-                "if(ISBLANK('Internet Sales'[Internet Current Quarter Gross Profit Performance]),BLANK(),\r",
+                "if(ISBLANK('Internet Sales'[Internet Current Quarter Gross Profit Performance]),BLANK(),",
-                "                    If('Internet Sales'[Internet Current Quarter Gross Profit Performance]<0.8,-1,\r",
+                "                    If('Internet Sales'[Internet Current Quarter Gross Profit Performance]<0.8,-1,",
-                "\t                If('Internet Sales'[Internet Current Quarter Gross Profit Performance]<1.03,0,1)\r",
+                "\t                If('Internet Sales'[Internet Current Quarter Gross Profit Performance]<1.03,0,1)",
-                "                )\r",
+                "                )",
-                "            )\r",
+                "            )",
                 "            "
               ],
               "annotations": [
@@ -4217,11 +4216,11 @@
               "targetExpression": "1.1",
               "statusGraphic": "Three Symbols UnCircled Colored",
               "statusExpression": [
-                "if(ISBLANK('Reseller Sales'[Reseller Current Quarter Sales Performance]),BLANK(),\r",
+                "if(ISBLANK('Reseller Sales'[Reseller Current Quarter Sales Performance]),BLANK(),",
-                "                    If('Reseller Sales'[Reseller Current Quarter Sales Performance]<0.8,-1,\r",
+                "                    If('Reseller Sales'[Reseller Current Quarter Sales Performance]<0.8,-1,",
-                "\t                If('Reseller Sales'[Reseller Current Quarter Sales Performance]<1.07,0,1)\r",
+                "\t                If('Reseller Sales'[Reseller Current Quarter Sales Performance]<1.07,0,1)",
-                "                )\r",
+                "                )",
-                "            )\r",
+                "            )",
                 "            "
               ],
               "annotations": [
@@ -4263,11 +4262,11 @@
               "targetExpression": "1.25",
               "statusGraphic": "Three Symbols UnCircled Colored",
               "statusExpression": [
-                "if(ISBLANK('Reseller Sales'[Reseller Current Quarter Gross Profit Performance]),BLANK(),\r",
+                "if(ISBLANK('Reseller Sales'[Reseller Current Quarter Gross Profit Performance]),BLANK(),",
-                "                    If('Reseller Sales'[Reseller Current Quarter Gross Profit Performance]<0.8,-1,\r",
+                "                    If('Reseller Sales'[Reseller Current Quarter Gross Profit Performance]<0.8,-1,",
-                "\t                If('Reseller Sales'[Reseller Current Quarter Gross Profit Performance]<1.03,0,1)\r",
+                "\t                If('Reseller Sales'[Reseller Current Quarter Gross Profit Performance]<1.03,0,1)",
-                "                )\r",
+                "                )",
-                "            )\r",
+                "            )",
                 "            "
               ],
               "annotations": [
@@ -6005,5 +6004,6 @@
         "modelPermission": "refresh"
       }
     ]
-  }
+  },
+  "id": "SemanticModel"
 }

AsPartitionProcessing/AsPartitionProcessing.SampleClient/Program.cs

@@ -151,9 +151,12 @@ namespace AsPartitionProcessing.SampleClient
                                 granularity: Granularity.Monthly,
                                 numberOfPartitionsFull: 12,
                                 numberOfPartitionsForIncrementalProcess: 3,
+                                maxDateIsNow: false,
                                 maxDate: Convert.ToDateTime("2012-12-01"),
-                                sourceTableName: "[dbo].[FactInternetSales]",
+                                integerDateKey: true,
-                                sourcePartitionColumn: "OrderDateKey"
+                                templateSourceQuery: "SELECT * FROM [dbo].[FactInternetSales] " +
+                                    "WHERE OrderDateKey >= {0} AND OrderDateKey < {1} " +
+                                    "ORDER BY OrderDateKey"
                             )
                         }
                     ),
@@ -168,9 +171,12 @@ namespace AsPartitionProcessing.SampleClient
                                 granularity: Granularity.Yearly,
                                 numberOfPartitionsFull: 3,
                                 numberOfPartitionsForIncrementalProcess: 1,
+                                maxDateIsNow: false,
                                 maxDate: Convert.ToDateTime("2012-12-01"),
-                                sourceTableName: "[dbo].[FactResellerSales]",
+                                integerDateKey: true,
-                                sourcePartitionColumn: "OrderDateKey"
+                                templateSourceQuery: "SELECT * FROM [dbo].[FactResellerSales] " +
+                                    "WHERE OrderDateKey >= {0} AND OrderDateKey < {1} " +
+                                    "ORDER BY OrderDateKey"
                             )
                         }
                     )

AsPartitionProcessing/AsPartitionProcessing/ConfigDatabaseHelper.cs

@@ -40,9 +40,10 @@ namespace AsPartitionProcessing
                               ,[Granularity]
                               ,[NumberOfPartitionsFull]
                               ,[NumberOfPartitionsForIncrementalProcess]
+                              ,[MaxDateIsNow]
                               ,[MaxDate]
-                              ,[SourceTableName]
+                              ,[IntegerDateKey]
-                              ,[SourcePartitionColumn]
+                              ,[TemplateSourceQuery]
                           FROM [dbo].[vPartitioningConfiguration]
                           WHERE [DoNotProcess] = 0
                           ORDER BY
@@ -98,9 +99,10 @@ namespace AsPartitionProcessing
                                     (Granularity)Convert.ToInt32(reader["Granularity"]),
                                     Convert.ToInt32(reader["NumberOfPartitionsFull"]),
                                     Convert.ToInt32(reader["NumberOfPartitionsForIncrementalProcess"]),
-                                    Convert.ToDateTime(reader["MaxDate"]),
+                                    Convert.ToBoolean(reader["MaxDateIsNow"]),
-                                    Convert.ToString(reader["SourceTableName"]),
+                                    (reader["MaxDate"] == DBNull.Value ? DateTime.MinValue : Convert.ToDateTime(reader["MaxDate"])),
-                                    Convert.ToString(reader["SourcePartitionColumn"])
+                                    Convert.ToBoolean(reader["IntegerDateKey"]),
+                                    Convert.ToString(reader["TemplateSourceQuery"])
                                 )
                             );
                         }

AsPartitionProcessing/AsPartitionProcessing/PartitionProcessor.cs

@@ -164,7 +164,7 @@ namespace AsPartitionProcessing
                         //Ensure template partition doesn't contain any data
                         if (_modelConfiguration.InitialSetUp)
                         {
-                            ((QueryPartitionSource)templatePartition.Source).Query = String.Format("SELECT * FROM {0} WHERE 0=1", tableConfiguration.PartitioningConfigurations[0].SourceTableName); //assuming the same for all partitioning configurations
+                            ((QueryPartitionSource)templatePartition.Source).Query = String.Format(tableConfiguration.PartitioningConfigurations[0].TemplateSourceQuery, GetDateKey("19010102", tableConfiguration.PartitioningConfigurations[0], false), GetDateKey("19010101", tableConfiguration.PartitioningConfigurations[0], false)); //Query generated will always return nothing
                             templatePartition.RequestRefresh(RefreshType.DataOnly);
                         }
                     }
@@ -436,6 +436,47 @@ namespace AsPartitionProcessing
 
         #region Private Methods
 
+        private static string GetDateKey(string partitionKey, PartitioningConfiguration partitioningConfiguration, bool addPeriod)
+        {
+            DateTime dateVal = new DateTime();
+
+            switch (partitioningConfiguration.Granularity)
+            {
+                case Granularity.Daily:
+                    dateVal = new DateTime(Convert.ToInt32(partitionKey.Substring(0, 4)), Convert.ToInt32(partitionKey.Substring(4, 2)), Convert.ToInt32(partitionKey.Substring(6, 2)));
+                    if (addPeriod)
+                    {
+                        dateVal = dateVal.AddDays(1);
+                    }
+                    break;
+                case Granularity.Monthly:
+                    dateVal = new DateTime(Convert.ToInt32(partitionKey.Substring(0, 4)), Convert.ToInt32(partitionKey.Substring(4, 2)), 1);
+                    if (addPeriod)
+                    {
+                        dateVal = dateVal.AddMonths(1);
+                    }
+                    break;
+                case Granularity.Yearly:
+                    dateVal = new DateTime(Convert.ToInt32(partitionKey.Substring(0, 4)), 1, 1);
+                    if (addPeriod)
+                    {
+                        dateVal = dateVal.AddYears(1);
+                    }
+                    break;
+                default:
+                    break;
+            }
+
+            if (partitioningConfiguration.IntegerDateKey)
+            {
+                return dateVal.ToString("yyyyMMdd");
+            }
+            else
+            {
+                return $"'{dateVal.ToString("yyyy-MM-dd")}'";
+            }
+        }
+
         private static void IncrementalProcessPartition(string partitionKey, Partition partitionToProcess, Granularity granularity)
         {
             if (_modelConfiguration.IncrementalOnline)
@@ -579,28 +620,14 @@ namespace AsPartitionProcessing
 
         private static Partition CreateNewPartition(Table table, Partition templatePartition, PartitioningConfiguration partitioningConfiguration, string partitionKey)
         {
-            string selectQueryTemplate;
+            string beginParam = GetDateKey(partitionKey, partitioningConfiguration, false);
-            switch (partitioningConfiguration.Granularity)
+            string endParam = GetDateKey(partitionKey, partitioningConfiguration, true);
-            {
-                //Format that might work on more data sources, but requires flag to indicate whether partitioning column is date or integer YYYYMMDD or not:
-                //               SELECT	YEAR(CURRENT_TIMESTAMP) * 10000 + MONTH(CURRENT_TIMESTAMP) * 100 + DAY(CURRENT_TIMESTAMP)
-                //ANSI standard to get month from date is EXTRACT(MONTH FROM @DateTimeVarUnclean), which doesn't work with SQL Server
 
-                case Granularity.Daily:
-                    selectQueryTemplate = "SELECT * FROM {0} WHERE CAST(CONVERT(varchar, {1}, 112) AS int) = {2} ORDER BY {1}";
-                    break;
-                case Granularity.Monthly:
-                    selectQueryTemplate = "SELECT * FROM {0} WHERE FLOOR(CAST(CONVERT(varchar, {1}, 112) AS int) / 100) = {2} ORDER BY {1}";
-                    break;
-                default: //Granularity.Yearly:
-                    selectQueryTemplate = "SELECT * FROM {0} WHERE FLOOR(CAST(CONVERT(varchar, {1}, 112) AS int) / 10000) = {2} ORDER BY {1}";
-                    break;
-            }
             Partition newPartition;
             newPartition = new Partition();
             templatePartition.CopyTo(newPartition);
             newPartition.Name = partitionKey;
-            ((QueryPartitionSource)newPartition.Source).Query = String.Format(selectQueryTemplate, partitioningConfiguration.SourceTableName, partitioningConfiguration.SourcePartitionColumn, partitionKey);
+            ((QueryPartitionSource)newPartition.Source).Query = String.Format(partitioningConfiguration.TemplateSourceQuery, beginParam, endParam);
             table.Partitions.Add(newPartition);
             return newPartition;
         }

AsPartitionProcessing/AsPartitionProcessing/PartitioningConfiguration.cs

@@ -28,7 +28,7 @@ namespace AsPartitionProcessing
         public int NumberOfPartitionsForIncrementalProcess { get; set; }
 
         /// <summary>
-        /// The maximum date that needs to be accounted for in the partitioned table.
+        /// If MaxDateIsNow = false, the maximum date that needs to be accounted for in the partitioned table.
         /// </summary>
         public DateTime MaxDate { get; set; }
 
@@ -43,14 +43,14 @@ namespace AsPartitionProcessing
         public DateTime UpperBoundary { get; }
 
         /// <summary>
-        /// Name of the source table in the relational database.
+        /// Assumes date keys are integers. If false assumes date type.
         /// </summary>
-        public string SourceTableName { get; set; }
+        public bool IntegerDateKey { get; set; }
 
         /// <summary>
-        /// Name of the source column from the table in the relational database.
+        /// Template query used for partition source queries.
         /// </summary>
-        public string SourcePartitionColumn { get; set; }
+        public string TemplateSourceQuery { get; set; }
 
         /// <summary>
         /// Initialize partitioning configuration for partitioned table. Normally populated from a configuration database.
@@ -60,48 +60,58 @@ namespace AsPartitionProcessing
         /// <param name="granularity">Partition granularity, which can be Yearly, Monthly or Daily.</param>
         /// <param name="numberOfPartitionsFull">Count of all partitions in the rolling window. For example, a rolling window of 10 years partitioned by month would result in 120 partitions.</param>
         /// <param name="numberOfPartitionsForIncrementalProcess">Count of “hot partitions” where the data can change. For example, it may be necessary to refresh the most recent 3 months of data every day. This only applies to the most recent partitions.</param>
+        /// <param name="maxDateIsNow">Assumes maximum date to be accounted for is today.</param>
         /// <param name="maxDate">The maximum date that needs to be accounted for in the partitioned table. Represents the upper boundary of the rolling window.</param>
-        /// <param name="sourceTableName">Name of the source table in the relational database.</param>
+        /// <param name="integerDateKey">Assumes date keys are integers. If false assumes date type.</param>
-        /// <param name="sourcePartitionColumn">Name of the source column from the table in the relational database.</param>
+        /// <param name="templateSourceQuery">Template query used for partition source queries.</param>
         public PartitioningConfiguration(
             int partitioningConfigurationID,
             Granularity granularity,
             int numberOfPartitionsFull,
             int numberOfPartitionsForIncrementalProcess,
+            bool maxDateIsNow,
             DateTime maxDate,
-            string sourceTableName,
+            bool integerDateKey,
-            string sourcePartitionColumn
+            string templateSourceQuery
             )
         {
             PartitioningConfigurationID = partitioningConfigurationID;
             Granularity = granularity;
             NumberOfPartitionsFull = numberOfPartitionsFull;
             NumberOfPartitionsForIncrementalProcess = numberOfPartitionsForIncrementalProcess;
+            if (maxDateIsNow)
+            {
+                MaxDate = DateTime.Today;
+            }
+            else
+            {
                 MaxDate = maxDate;
-            SourceTableName = sourceTableName;
+            }
-            SourcePartitionColumn = sourcePartitionColumn;
+            IntegerDateKey = integerDateKey;
+            TemplateSourceQuery = templateSourceQuery;
 
             switch (granularity)
             {
                 case Granularity.Daily:
-                    LowerBoundary = maxDate.AddDays(-numberOfPartitionsFull + 1);
+                    LowerBoundary = MaxDate.AddDays(-numberOfPartitionsFull + 1);
-                    UpperBoundary = maxDate;
+                    UpperBoundary = MaxDate;
                     break;
 
                 case Granularity.Monthly:
-                    LowerBoundary = Convert.ToDateTime(maxDate.AddMonths(-numberOfPartitionsFull + 1).ToString("yyyy-MMM-01"));
+                    LowerBoundary = Convert.ToDateTime(MaxDate.AddMonths(-numberOfPartitionsFull + 1).ToString("yyyy-MMM-01"));
-                    UpperBoundary = Convert.ToDateTime(maxDate.AddMonths(1).ToString("yyyy-MMM-01")).AddDays(-1);
+                    UpperBoundary = Convert.ToDateTime(MaxDate.AddMonths(1).ToString("yyyy-MMM-01")).AddDays(-1);
                     break;
 
                 case Granularity.Yearly:
-                    LowerBoundary = Convert.ToDateTime(maxDate.AddYears(-numberOfPartitionsFull + 1).ToString("yyyy-01-01"));
+                    LowerBoundary = Convert.ToDateTime(MaxDate.AddYears(-numberOfPartitionsFull + 1).ToString("yyyy-01-01"));
-                    UpperBoundary = Convert.ToDateTime(maxDate.AddYears(1).ToString("yyyy-01-01")).AddDays(-1);
+                    UpperBoundary = Convert.ToDateTime(MaxDate.AddYears(1).ToString("yyyy-01-01")).AddDays(-1);
                     break;
 
                 default:
                     break;
             }
         }
+
         public int CompareTo(PartitioningConfiguration other) => string.Compare(this.LowerBoundary.ToString("yyyy-MM-dd"), other.LowerBoundary.ToString("yyyy-MM-dd"));
     }
 

AsPartitionProcessing/CreateDatabaseObjects.sql

@@ -25,9 +25,10 @@ CREATE TABLE [dbo].[PartitioningConfiguration](
     [Granularity] [tinyint] NOT NULL,
     [NumberOfPartitionsFull] [int] NOT NULL,
     [NumberOfPartitionsForIncrementalProcess] [int] NOT NULL,
-    [MaxDate] [date] NOT NULL,
+    [MaxDateIsNow] [bit] NOT NULL,
-    [SourceTableName] [varchar](255) NOT NULL,
+    [MaxDate] [date] NULL,
-    [SourcePartitionColumn] [varchar](255) NOT NULL,
+    [IntegerDateKey] [bit] NOT NULL,
+    [TemplateSourceQuery] [varchar](max) NOT NULL,
  CONSTRAINT [PK_PartitioningConfiguration] PRIMARY KEY CLUSTERED 
 (
     [PartitioningConfigurationID] ASC
@@ -119,9 +120,10 @@ SELECT m.[ModelConfigurationID]
       ,p.[Granularity]
       ,p.[NumberOfPartitionsFull]
       ,p.[NumberOfPartitionsForIncrementalProcess]
+	  ,p.[MaxDateIsNow]
       ,p.[MaxDate]
-      ,p.[SourceTableName]
+      ,p.[IntegerDateKey]
-      ,p.[SourcePartitionColumn]
+      ,p.[TemplateSourceQuery]
   FROM [dbo].[ModelConfiguration] m
 INNER JOIN [dbo].[TableConfiguration] t ON m.[ModelConfigurationID] = t.[ModelConfigurationID]
 LEFT JOIN [dbo].[PartitioningConfiguration] p ON t.[TableConfigurationID] = p.[TableConfigurationID]

AsPartitionProcessing/SampleConfiguration.sql

@@ -31,9 +31,10 @@ VALUES(
     ,1                          --[Granularity]   1=Monthly
     ,12                         --[NumberOfPartitionsFull]
     ,3                          --[NumberOfPartitionsForIncrementalProcess]
+    ,0                          --[MaxDateIsNow]
     ,'2012-12-01'               --[MaxDate]
-    ,'[dbo].[FactInternetSales]'--[SourceTableName]
+    ,1                          --[IntegerDateKey]
-    ,'OrderDateKey'             --[SourcePartitionColumn]
+    ,'SELECT * FROM [dbo].[FactInternetSales] WHERE OrderDateKey >= {0} AND OrderDateKey < {1} ORDER BY OrderDateKey' --[TemplateSourceQuery]
 ),
 (
      2                          --[PartitioningConfigurationID]
@@ -41,7 +42,8 @@ VALUES(
     ,2                          --[Granularity]   2=Yearly
     ,3                          --[NumberOfPartitionsFull]
     ,1                          --[NumberOfPartitionsForIncrementalProcess]
+    ,0                          --[MaxDateIsNow]
     ,'2012-12-01'               --[MaxDate]
-    ,'[dbo].[FactResellerSales]'--[SourceTableName]
+    ,1                          --[IntegerDateKey]
-    ,'OrderDateKey'             --[SourcePartitionColumn]
+    ,'SELECT * FROM [dbo].[FactResellerSales] WHERE OrderDateKey >= {0} AND OrderDateKey < {1} ORDER BY OrderDateKey' --[TemplateSourceQuery]
 );