Generic queries for different data sources
This commit is contained in:
christianwade
parent
9a8a01a901
commit
1247a48aa3
@ -1,6 +1,5 @@
|
||||
{
|
||||
"name": "SemanticModel",
|
||||
"id": "SemanticModel",
|
||||
"compatibilityLevel": 1200,
|
||||
"model": {
|
||||
"culture": "en-US",
|
||||
@ -3643,11 +3642,11 @@
|
||||
"targetExpression": "1.1",
|
||||
"statusGraphic": "Three Symbols UnCircled Colored",
|
||||
"statusExpression": [
|
||||
"if(ISBLANK('Internet Sales'[Internet Current Quarter Sales Performance]),BLANK(),\r",
|
||||
" If('Internet Sales'[Internet Current Quarter Sales Performance]<1,-1,\r",
|
||||
"\t If('Internet Sales'[Internet Current Quarter Sales Performance]<1.07,0,1)\r",
|
||||
" )\r",
|
||||
" )\r",
|
||||
"if(ISBLANK('Internet Sales'[Internet Current Quarter Sales Performance]),BLANK(),",
|
||||
" If('Internet Sales'[Internet Current Quarter Sales Performance]<1,-1,",
|
||||
"\t If('Internet Sales'[Internet Current Quarter Sales Performance]<1.07,0,1)",
|
||||
" )",
|
||||
" )",
|
||||
" "
|
||||
],
|
||||
"annotations": [
|
||||
@ -3744,11 +3743,11 @@
|
||||
"targetExpression": "1.25",
|
||||
"statusGraphic": "Three Symbols UnCircled Colored",
|
||||
"statusExpression": [
|
||||
"if(ISBLANK('Internet Sales'[Internet Current Quarter Gross Profit Performance]),BLANK(),\r",
|
||||
" If('Internet Sales'[Internet Current Quarter Gross Profit Performance]<0.8,-1,\r",
|
||||
"\t If('Internet Sales'[Internet Current Quarter Gross Profit Performance]<1.03,0,1)\r",
|
||||
" )\r",
|
||||
" )\r",
|
||||
"if(ISBLANK('Internet Sales'[Internet Current Quarter Gross Profit Performance]),BLANK(),",
|
||||
" If('Internet Sales'[Internet Current Quarter Gross Profit Performance]<0.8,-1,",
|
||||
"\t If('Internet Sales'[Internet Current Quarter Gross Profit Performance]<1.03,0,1)",
|
||||
" )",
|
||||
" )",
|
||||
" "
|
||||
],
|
||||
"annotations": [
|
||||
@ -4217,11 +4216,11 @@
|
||||
"targetExpression": "1.1",
|
||||
"statusGraphic": "Three Symbols UnCircled Colored",
|
||||
"statusExpression": [
|
||||
"if(ISBLANK('Reseller Sales'[Reseller Current Quarter Sales Performance]),BLANK(),\r",
|
||||
" If('Reseller Sales'[Reseller Current Quarter Sales Performance]<0.8,-1,\r",
|
||||
"\t If('Reseller Sales'[Reseller Current Quarter Sales Performance]<1.07,0,1)\r",
|
||||
" )\r",
|
||||
" )\r",
|
||||
"if(ISBLANK('Reseller Sales'[Reseller Current Quarter Sales Performance]),BLANK(),",
|
||||
" If('Reseller Sales'[Reseller Current Quarter Sales Performance]<0.8,-1,",
|
||||
"\t If('Reseller Sales'[Reseller Current Quarter Sales Performance]<1.07,0,1)",
|
||||
" )",
|
||||
" )",
|
||||
" "
|
||||
],
|
||||
"annotations": [
|
||||
@ -4263,11 +4262,11 @@
|
||||
"targetExpression": "1.25",
|
||||
"statusGraphic": "Three Symbols UnCircled Colored",
|
||||
"statusExpression": [
|
||||
"if(ISBLANK('Reseller Sales'[Reseller Current Quarter Gross Profit Performance]),BLANK(),\r",
|
||||
" If('Reseller Sales'[Reseller Current Quarter Gross Profit Performance]<0.8,-1,\r",
|
||||
"\t If('Reseller Sales'[Reseller Current Quarter Gross Profit Performance]<1.03,0,1)\r",
|
||||
" )\r",
|
||||
" )\r",
|
||||
"if(ISBLANK('Reseller Sales'[Reseller Current Quarter Gross Profit Performance]),BLANK(),",
|
||||
" If('Reseller Sales'[Reseller Current Quarter Gross Profit Performance]<0.8,-1,",
|
||||
"\t If('Reseller Sales'[Reseller Current Quarter Gross Profit Performance]<1.03,0,1)",
|
||||
" )",
|
||||
" )",
|
||||
" "
|
||||
],
|
||||
"annotations": [
|
||||
@ -6005,5 +6004,6 @@
|
||||
"modelPermission": "refresh"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"id": "SemanticModel"
|
||||
}
|
||||
@ -151,9 +151,12 @@ namespace AsPartitionProcessing.SampleClient
|
||||
granularity: Granularity.Monthly,
|
||||
numberOfPartitionsFull: 12,
|
||||
numberOfPartitionsForIncrementalProcess: 3,
|
||||
maxDateIsNow: false,
|
||||
maxDate: Convert.ToDateTime("2012-12-01"),
|
||||
sourceTableName: "[dbo].[FactInternetSales]",
|
||||
sourcePartitionColumn: "OrderDateKey"
|
||||
integerDateKey: true,
|
||||
templateSourceQuery: "SELECT * FROM [dbo].[FactInternetSales] " +
|
||||
"WHERE OrderDateKey >= {0} AND OrderDateKey < {1} " +
|
||||
"ORDER BY OrderDateKey"
|
||||
)
|
||||
}
|
||||
),
|
||||
@ -168,9 +171,12 @@ namespace AsPartitionProcessing.SampleClient
|
||||
granularity: Granularity.Yearly,
|
||||
numberOfPartitionsFull: 3,
|
||||
numberOfPartitionsForIncrementalProcess: 1,
|
||||
maxDateIsNow: false,
|
||||
maxDate: Convert.ToDateTime("2012-12-01"),
|
||||
sourceTableName: "[dbo].[FactResellerSales]",
|
||||
sourcePartitionColumn: "OrderDateKey"
|
||||
integerDateKey: true,
|
||||
templateSourceQuery: "SELECT * FROM [dbo].[FactResellerSales] " +
|
||||
"WHERE OrderDateKey >= {0} AND OrderDateKey < {1} " +
|
||||
"ORDER BY OrderDateKey"
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
@ -40,9 +40,10 @@ namespace AsPartitionProcessing
|
||||
,[Granularity]
|
||||
,[NumberOfPartitionsFull]
|
||||
,[NumberOfPartitionsForIncrementalProcess]
|
||||
,[MaxDateIsNow]
|
||||
,[MaxDate]
|
||||
,[SourceTableName]
|
||||
,[SourcePartitionColumn]
|
||||
,[IntegerDateKey]
|
||||
,[TemplateSourceQuery]
|
||||
FROM [dbo].[vPartitioningConfiguration]
|
||||
WHERE [DoNotProcess] = 0
|
||||
ORDER BY
|
||||
@ -98,9 +99,10 @@ namespace AsPartitionProcessing
|
||||
(Granularity)Convert.ToInt32(reader["Granularity"]),
|
||||
Convert.ToInt32(reader["NumberOfPartitionsFull"]),
|
||||
Convert.ToInt32(reader["NumberOfPartitionsForIncrementalProcess"]),
|
||||
Convert.ToDateTime(reader["MaxDate"]),
|
||||
Convert.ToString(reader["SourceTableName"]),
|
||||
Convert.ToString(reader["SourcePartitionColumn"])
|
||||
Convert.ToBoolean(reader["MaxDateIsNow"]),
|
||||
(reader["MaxDate"] == DBNull.Value ? DateTime.MinValue : Convert.ToDateTime(reader["MaxDate"])),
|
||||
Convert.ToBoolean(reader["IntegerDateKey"]),
|
||||
Convert.ToString(reader["TemplateSourceQuery"])
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
@ -164,7 +164,7 @@ namespace AsPartitionProcessing
|
||||
//Ensure template partition doesn't contain any data
|
||||
if (_modelConfiguration.InitialSetUp)
|
||||
{
|
||||
((QueryPartitionSource)templatePartition.Source).Query = String.Format("SELECT * FROM {0} WHERE 0=1", tableConfiguration.PartitioningConfigurations[0].SourceTableName); //assuming the same for all partitioning configurations
|
||||
((QueryPartitionSource)templatePartition.Source).Query = String.Format(tableConfiguration.PartitioningConfigurations[0].TemplateSourceQuery, GetDateKey("19010102", tableConfiguration.PartitioningConfigurations[0], false), GetDateKey("19010101", tableConfiguration.PartitioningConfigurations[0], false)); //Query generated will always return nothing
|
||||
templatePartition.RequestRefresh(RefreshType.DataOnly);
|
||||
}
|
||||
}
|
||||
@ -436,6 +436,47 @@ namespace AsPartitionProcessing
|
||||
|
||||
#region Private Methods
|
||||
|
||||
private static string GetDateKey(string partitionKey, PartitioningConfiguration partitioningConfiguration, bool addPeriod)
|
||||
{
|
||||
DateTime dateVal = new DateTime();
|
||||
|
||||
switch (partitioningConfiguration.Granularity)
|
||||
{
|
||||
case Granularity.Daily:
|
||||
dateVal = new DateTime(Convert.ToInt32(partitionKey.Substring(0, 4)), Convert.ToInt32(partitionKey.Substring(4, 2)), Convert.ToInt32(partitionKey.Substring(6, 2)));
|
||||
if (addPeriod)
|
||||
{
|
||||
dateVal = dateVal.AddDays(1);
|
||||
}
|
||||
break;
|
||||
case Granularity.Monthly:
|
||||
dateVal = new DateTime(Convert.ToInt32(partitionKey.Substring(0, 4)), Convert.ToInt32(partitionKey.Substring(4, 2)), 1);
|
||||
if (addPeriod)
|
||||
{
|
||||
dateVal = dateVal.AddMonths(1);
|
||||
}
|
||||
break;
|
||||
case Granularity.Yearly:
|
||||
dateVal = new DateTime(Convert.ToInt32(partitionKey.Substring(0, 4)), 1, 1);
|
||||
if (addPeriod)
|
||||
{
|
||||
dateVal = dateVal.AddYears(1);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
if (partitioningConfiguration.IntegerDateKey)
|
||||
{
|
||||
return dateVal.ToString("yyyyMMdd");
|
||||
}
|
||||
else
|
||||
{
|
||||
return $"'{dateVal.ToString("yyyy-MM-dd")}'";
|
||||
}
|
||||
}
|
||||
|
||||
private static void IncrementalProcessPartition(string partitionKey, Partition partitionToProcess, Granularity granularity)
|
||||
{
|
||||
if (_modelConfiguration.IncrementalOnline)
|
||||
@ -579,28 +620,14 @@ namespace AsPartitionProcessing
|
||||
|
||||
private static Partition CreateNewPartition(Table table, Partition templatePartition, PartitioningConfiguration partitioningConfiguration, string partitionKey)
|
||||
{
|
||||
string selectQueryTemplate;
|
||||
switch (partitioningConfiguration.Granularity)
|
||||
{
|
||||
//Format that might work on more data sources, but requires flag to indicate whether partitioning column is date or integer YYYYMMDD or not:
|
||||
// SELECT YEAR(CURRENT_TIMESTAMP) * 10000 + MONTH(CURRENT_TIMESTAMP) * 100 + DAY(CURRENT_TIMESTAMP)
|
||||
//ANSI standard to get month from date is EXTRACT(MONTH FROM @DateTimeVarUnclean), which doesn't work with SQL Server
|
||||
string beginParam = GetDateKey(partitionKey, partitioningConfiguration, false);
|
||||
string endParam = GetDateKey(partitionKey, partitioningConfiguration, true);
|
||||
|
||||
case Granularity.Daily:
|
||||
selectQueryTemplate = "SELECT * FROM {0} WHERE CAST(CONVERT(varchar, {1}, 112) AS int) = {2} ORDER BY {1}";
|
||||
break;
|
||||
case Granularity.Monthly:
|
||||
selectQueryTemplate = "SELECT * FROM {0} WHERE FLOOR(CAST(CONVERT(varchar, {1}, 112) AS int) / 100) = {2} ORDER BY {1}";
|
||||
break;
|
||||
default: //Granularity.Yearly:
|
||||
selectQueryTemplate = "SELECT * FROM {0} WHERE FLOOR(CAST(CONVERT(varchar, {1}, 112) AS int) / 10000) = {2} ORDER BY {1}";
|
||||
break;
|
||||
}
|
||||
Partition newPartition;
|
||||
newPartition = new Partition();
|
||||
templatePartition.CopyTo(newPartition);
|
||||
newPartition.Name = partitionKey;
|
||||
((QueryPartitionSource)newPartition.Source).Query = String.Format(selectQueryTemplate, partitioningConfiguration.SourceTableName, partitioningConfiguration.SourcePartitionColumn, partitionKey);
|
||||
((QueryPartitionSource)newPartition.Source).Query = String.Format(partitioningConfiguration.TemplateSourceQuery, beginParam, endParam);
|
||||
table.Partitions.Add(newPartition);
|
||||
return newPartition;
|
||||
}
|
||||
|
||||
@ -28,7 +28,7 @@ namespace AsPartitionProcessing
|
||||
public int NumberOfPartitionsForIncrementalProcess { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The maximum date that needs to be accounted for in the partitioned table.
|
||||
/// If MaxDateIsNow = false, the maximum date that needs to be accounted for in the partitioned table.
|
||||
/// </summary>
|
||||
public DateTime MaxDate { get; set; }
|
||||
|
||||
@ -43,14 +43,14 @@ namespace AsPartitionProcessing
|
||||
public DateTime UpperBoundary { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Name of the source table in the relational database.
|
||||
/// Assumes date keys are integers. If false assumes date type.
|
||||
/// </summary>
|
||||
public string SourceTableName { get; set; }
|
||||
public bool IntegerDateKey { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Name of the source column from the table in the relational database.
|
||||
/// Template query used for partition source queries.
|
||||
/// </summary>
|
||||
public string SourcePartitionColumn { get; set; }
|
||||
public string TemplateSourceQuery { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Initialize partitioning configuration for partitioned table. Normally populated from a configuration database.
|
||||
@ -60,48 +60,58 @@ namespace AsPartitionProcessing
|
||||
/// <param name="granularity">Partition granularity, which can be Yearly, Monthly or Daily.</param>
|
||||
/// <param name="numberOfPartitionsFull">Count of all partitions in the rolling window. For example, a rolling window of 10 years partitioned by month would result in 120 partitions.</param>
|
||||
/// <param name="numberOfPartitionsForIncrementalProcess">Count of “hot partitions” where the data can change. For example, it may be necessary to refresh the most recent 3 months of data every day. This only applies to the most recent partitions.</param>
|
||||
/// <param name="maxDateIsNow">Assumes maximum date to be accounted for is today.</param>
|
||||
/// <param name="maxDate">The maximum date that needs to be accounted for in the partitioned table. Represents the upper boundary of the rolling window.</param>
|
||||
/// <param name="sourceTableName">Name of the source table in the relational database.</param>
|
||||
/// <param name="sourcePartitionColumn">Name of the source column from the table in the relational database.</param>
|
||||
/// <param name="integerDateKey">Assumes date keys are integers. If false assumes date type.</param>
|
||||
/// <param name="templateSourceQuery">Template query used for partition source queries.</param>
|
||||
public PartitioningConfiguration(
|
||||
int partitioningConfigurationID,
|
||||
Granularity granularity,
|
||||
int numberOfPartitionsFull,
|
||||
int numberOfPartitionsForIncrementalProcess,
|
||||
bool maxDateIsNow,
|
||||
DateTime maxDate,
|
||||
string sourceTableName,
|
||||
string sourcePartitionColumn
|
||||
bool integerDateKey,
|
||||
string templateSourceQuery
|
||||
)
|
||||
{
|
||||
PartitioningConfigurationID = partitioningConfigurationID;
|
||||
Granularity = granularity;
|
||||
NumberOfPartitionsFull = numberOfPartitionsFull;
|
||||
NumberOfPartitionsForIncrementalProcess = numberOfPartitionsForIncrementalProcess;
|
||||
MaxDate = maxDate;
|
||||
SourceTableName = sourceTableName;
|
||||
SourcePartitionColumn = sourcePartitionColumn;
|
||||
if (maxDateIsNow)
|
||||
{
|
||||
MaxDate = DateTime.Today;
|
||||
}
|
||||
else
|
||||
{
|
||||
MaxDate = maxDate;
|
||||
}
|
||||
IntegerDateKey = integerDateKey;
|
||||
TemplateSourceQuery = templateSourceQuery;
|
||||
|
||||
switch (granularity)
|
||||
{
|
||||
case Granularity.Daily:
|
||||
LowerBoundary = maxDate.AddDays(-numberOfPartitionsFull + 1);
|
||||
UpperBoundary = maxDate;
|
||||
LowerBoundary = MaxDate.AddDays(-numberOfPartitionsFull + 1);
|
||||
UpperBoundary = MaxDate;
|
||||
break;
|
||||
|
||||
case Granularity.Monthly:
|
||||
LowerBoundary = Convert.ToDateTime(maxDate.AddMonths(-numberOfPartitionsFull + 1).ToString("yyyy-MMM-01"));
|
||||
UpperBoundary = Convert.ToDateTime(maxDate.AddMonths(1).ToString("yyyy-MMM-01")).AddDays(-1);
|
||||
LowerBoundary = Convert.ToDateTime(MaxDate.AddMonths(-numberOfPartitionsFull + 1).ToString("yyyy-MMM-01"));
|
||||
UpperBoundary = Convert.ToDateTime(MaxDate.AddMonths(1).ToString("yyyy-MMM-01")).AddDays(-1);
|
||||
break;
|
||||
|
||||
case Granularity.Yearly:
|
||||
LowerBoundary = Convert.ToDateTime(maxDate.AddYears(-numberOfPartitionsFull + 1).ToString("yyyy-01-01"));
|
||||
UpperBoundary = Convert.ToDateTime(maxDate.AddYears(1).ToString("yyyy-01-01")).AddDays(-1);
|
||||
LowerBoundary = Convert.ToDateTime(MaxDate.AddYears(-numberOfPartitionsFull + 1).ToString("yyyy-01-01"));
|
||||
UpperBoundary = Convert.ToDateTime(MaxDate.AddYears(1).ToString("yyyy-01-01")).AddDays(-1);
|
||||
break;
|
||||
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
public int CompareTo(PartitioningConfiguration other) => string.Compare(this.LowerBoundary.ToString("yyyy-MM-dd"), other.LowerBoundary.ToString("yyyy-MM-dd"));
|
||||
}
|
||||
|
||||
|
||||
@ -25,9 +25,10 @@ CREATE TABLE [dbo].[PartitioningConfiguration](
|
||||
[Granularity] [tinyint] NOT NULL,
|
||||
[NumberOfPartitionsFull] [int] NOT NULL,
|
||||
[NumberOfPartitionsForIncrementalProcess] [int] NOT NULL,
|
||||
[MaxDate] [date] NOT NULL,
|
||||
[SourceTableName] [varchar](255) NOT NULL,
|
||||
[SourcePartitionColumn] [varchar](255) NOT NULL,
|
||||
[MaxDateIsNow] [bit] NOT NULL,
|
||||
[MaxDate] [date] NULL,
|
||||
[IntegerDateKey] [bit] NOT NULL,
|
||||
[TemplateSourceQuery] [varchar](max) NOT NULL,
|
||||
CONSTRAINT [PK_PartitioningConfiguration] PRIMARY KEY CLUSTERED
|
||||
(
|
||||
[PartitioningConfigurationID] ASC
|
||||
@ -119,9 +120,10 @@ SELECT m.[ModelConfigurationID]
|
||||
,p.[Granularity]
|
||||
,p.[NumberOfPartitionsFull]
|
||||
,p.[NumberOfPartitionsForIncrementalProcess]
|
||||
,p.[MaxDateIsNow]
|
||||
,p.[MaxDate]
|
||||
,p.[SourceTableName]
|
||||
,p.[SourcePartitionColumn]
|
||||
,p.[IntegerDateKey]
|
||||
,p.[TemplateSourceQuery]
|
||||
FROM [dbo].[ModelConfiguration] m
|
||||
INNER JOIN [dbo].[TableConfiguration] t ON m.[ModelConfigurationID] = t.[ModelConfigurationID]
|
||||
LEFT JOIN [dbo].[PartitioningConfiguration] p ON t.[TableConfigurationID] = p.[TableConfigurationID]
|
||||
|
||||
@ -31,9 +31,10 @@ VALUES(
|
||||
,1 --[Granularity] 1=Monthly
|
||||
,12 --[NumberOfPartitionsFull]
|
||||
,3 --[NumberOfPartitionsForIncrementalProcess]
|
||||
,0 --[MaxDateIsNow]
|
||||
,'2012-12-01' --[MaxDate]
|
||||
,'[dbo].[FactInternetSales]'--[SourceTableName]
|
||||
,'OrderDateKey' --[SourcePartitionColumn]
|
||||
,1 --[IntegerDateKey]
|
||||
,'SELECT * FROM [dbo].[FactInternetSales] WHERE OrderDateKey >= {0} AND OrderDateKey < {1} ORDER BY OrderDateKey' --[TemplateSourceQuery]
|
||||
),
|
||||
(
|
||||
2 --[PartitioningConfigurationID]
|
||||
@ -41,7 +42,8 @@ VALUES(
|
||||
,2 --[Granularity] 2=Yearly
|
||||
,3 --[NumberOfPartitionsFull]
|
||||
,1 --[NumberOfPartitionsForIncrementalProcess]
|
||||
,0 --[MaxDateIsNow]
|
||||
,'2012-12-01' --[MaxDate]
|
||||
,'[dbo].[FactResellerSales]'--[SourceTableName]
|
||||
,'OrderDateKey' --[SourcePartitionColumn]
|
||||
,1 --[IntegerDateKey]
|
||||
,'SELECT * FROM [dbo].[FactResellerSales] WHERE OrderDateKey >= {0} AND OrderDateKey < {1} ORDER BY OrderDateKey' --[TemplateSourceQuery]
|
||||
);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user