From 21b7d4a90db87dc7fc3bef6388797b585bb2ebcb Mon Sep 17 00:00:00 2001 From: Preston Hervey Date: Mon, 17 Dec 2018 14:43:12 -0600 Subject: [PATCH] Added documentation and more example configuration files. --- .../AdventureWorksAggs.smproj | 2 +- .../AdventureWorksAggsProvider.smproj | 2 +- SetUpAggs/AggsConfig.json | 57 ++--------- SetUpAggs/AggsConfig_SetAggs1.json | 69 +++++++++++++ ...sConfig2.json => AggsConfig_SetAggs2.json} | 6 -- SetUpAggs/AggsConfig_SetDQ.json | 31 ++++++ SetUpAggs/README.md | 96 +++++++++++++++++++ SetUpAggs/SetUpAggs.csproj | 32 +++++++ 8 files changed, 238 insertions(+), 57 deletions(-) create mode 100644 SetUpAggs/AggsConfig_SetAggs1.json rename SetUpAggs/{AggsConfig2.json => AggsConfig_SetAggs2.json} (89%) create mode 100644 SetUpAggs/AggsConfig_SetDQ.json create mode 100644 SetUpAggs/README.md diff --git a/SetUpAggs/AW Internet Sales/AdventureWorksAggs.smproj b/SetUpAggs/AW Internet Sales/AdventureWorksAggs.smproj index 963435a..59b4ba8 100644 --- a/SetUpAggs/AW Internet Sales/AdventureWorksAggs.smproj +++ b/SetUpAggs/AW Internet Sales/AdventureWorksAggs.smproj @@ -24,7 +24,7 @@ Default - asazure://aspaastest14.asazure-int.windows.net/sukanyasaltsrc1 + localhost\SQL2016 Developer Unknown AdventureWorksAggs diff --git a/SetUpAggs/AdventureWorksAggsProvider/AdventureWorksAggsProvider.smproj b/SetUpAggs/AdventureWorksAggsProvider/AdventureWorksAggsProvider.smproj index 13c4019..27f7cc4 100644 --- a/SetUpAggs/AdventureWorksAggsProvider/AdventureWorksAggsProvider.smproj +++ b/SetUpAggs/AdventureWorksAggsProvider/AdventureWorksAggsProvider.smproj @@ -24,7 +24,7 @@ Default - asazure://aspaastest14.asazure-int.windows.net/sukanyasaltsrc1 + localhost\SQL2016 Developer Unknown AdventureWorksAggsProvider diff --git a/SetUpAggs/AggsConfig.json b/SetUpAggs/AggsConfig.json index 3460c5e..893c7c1 100644 --- a/SetUpAggs/AggsConfig.json +++ b/SetUpAggs/AggsConfig.json @@ -4,70 +4,29 @@ "tables": [ { "name": "DimGeography", - "mode": "Dual" + "mode": "DirectQuery" }, { "name": "DimCustomer", - "mode": "Dual" + "mode": "DirectQuery" }, { "name": "DimDate", - "mode": "Dual" + "mode": "DirectQuery" }, { +Products +Getting things ready ... "name": "DimProductSubcategory", - "mode": "Dual" + "mode": "DirectQuery" }, { "name": "DimProductCategory", - "mode": "Dual" + "mode": "DirectQuery" }, { "name": "FactInternetSalesAgg", - "mode": "Import", - "aggregationRules": [ - { - "aggTableColumn": "OrderDateKey", - "summarization": "GroupBy", - "detailTable": "FactInternetSales", - "detailTableColumn": "OrderDateKey" - }, - { - "aggTableColumn": "CustomerKey", - "summarization": "GroupBy", - "detailTable": "FactInternetSales", - "detailTableColumn": "CustomerKey" - }, - { - "aggTableColumn": "ProductSubcategoryKey", - "summarization": "GroupBy", - "detailTable": "DimProduct", - "detailTableColumn": "ProductSubcategoryKey" - }, - { - "aggTableColumn": "SalesAmount_Sum", - "summarization": "Sum", - "detailTable": "FactInternetSales", - "detailTableColumn": "SalesAmount" - }, - { - "aggTableColumn": "UnitPrice_Sum", - "summarization": "Min", - "detailTable": "FactInternetSales", - "detailTableColumn": "UnitPrice" - }, - { - "aggTableColumn": "UnitPrice_Count", - "summarization": "Count", - "detailTable": "FactInternetSales", - "detailTableColumn": "UnitPrice" - }, - { - "aggTableColumn": "FactInternetSales_Count", - "summarization": "CountTableRows", - "detailTable": "FactInternetSales" - } - ] + "mode": "DirectQuery" } ] } diff --git a/SetUpAggs/AggsConfig_SetAggs1.json b/SetUpAggs/AggsConfig_SetAggs1.json new file mode 100644 index 0000000..72c6cf7 --- /dev/null +++ b/SetUpAggs/AggsConfig_SetAggs1.json @@ -0,0 +1,69 @@ +{ + "database": { + "name": "AdventureWorksAggsProvider_PH", + "tables": [ + { + "name": "DimGeography", + "mode": "Dual" + }, + { + "name": "DimCustomer", + "mode": "Dual" + }, + { + "name": "DimDate", + "mode": "Dual" + }, + { + "name": "DimProductSubcategory", + "mode": "Dual" + }, + { + "name": "DimProductCategory", + "mode": "Dual" + }, + { + "name": "FactInternetSalesAgg", + "mode": "Import", + "aggregationRules": [ + { + "aggTableColumn": "OrderDateKey", + "summarization": "GroupBy", + "detailTable": "FactInternetSales", + "detailTableColumn": "OrderDateKey" + }, + { + "aggTableColumn": "CustomerKey", + "summarization": "GroupBy", + "detailTable": "FactInternetSales", + "detailTableColumn": "CustomerKey" + }, + { + "aggTableColumn": "ProductSubcategoryKey", + "summarization": "GroupBy", + "detailTable": "DimProduct", + "detailTableColumn": "ProductSubcategoryKey" + }, + { + "aggTableColumn": "SalesAmount_Sum", + "summarization": "Sum", + "detailTable": "FactInternetSales", + "detailTableColumn": "SalesAmount" + }, + { + "aggTableColumn": "UnitPrice_Sum", + "summarization": "Min", + "detailTable": "FactInternetSales", + "detailTableColumn": "UnitPrice" + }, + { + "aggTableColumn": "UnitPrice_Count", + "summarization": "Count", + "detailTable": "FactInternetSales", + "detailTableColumn": "UnitPrice" + } + ] + } + ] + } +} \ No newline at end of file diff --git a/SetUpAggs/AggsConfig2.json b/SetUpAggs/AggsConfig_SetAggs2.json similarity index 89% rename from SetUpAggs/AggsConfig2.json rename to SetUpAggs/AggsConfig_SetAggs2.json index 0c40309..eec08c1 100644 --- a/SetUpAggs/AggsConfig2.json +++ b/SetUpAggs/AggsConfig_SetAggs2.json @@ -55,12 +55,6 @@ "summarization": "Sum", "detailTable": "FactInternetSales", "detailTableColumn": "UnitPrice" - }, - { - "aggTableColumn": "UnitPrice_Count", - "summarization": "Count", - "detailTable": "FactInternetSales", - "detailTableColumn": "UnitPrice" }, { "aggTableColumn": "FactInternetSales_Count", diff --git a/SetUpAggs/AggsConfig_SetDQ.json b/SetUpAggs/AggsConfig_SetDQ.json new file mode 100644 index 0000000..f2195c8 --- /dev/null +++ b/SetUpAggs/AggsConfig_SetDQ.json @@ -0,0 +1,31 @@ +{ + "database": { + "name": "AdventureWorksAggsProvider_PH", + "tables": [ + { + "name": "DimGeography", + "mode": "DirectQuery" + }, + { + "name": "DimCustomer", + "mode": "DirectQuery" + }, + { + "name": "DimDate", + "mode": "DirectQuery" + }, + { + "name": "DimProductSubcategory", + "mode": "DirectQuery" + }, + { + "name": "DimProductCategory", + "mode": "DirectQuery" + }, + { + "name": "FactInternetSalesAgg", + "mode": "DirectQuery" + } + ] + } +} \ No newline at end of file diff --git a/SetUpAggs/README.md b/SetUpAggs/README.md new file mode 100644 index 0000000..43f80e9 --- /dev/null +++ b/SetUpAggs/README.md @@ -0,0 +1,96 @@ +# SetUpAggs + +This utility is designed to assist in updating an Analysis Services and/or Power BI data model to utilize the Composite Model and Aggregations features. The utility will connect to a specified Azure Analysis Services and/or Power BI instance and read in a configuration file, this configuration file will specify the database and associated tables to update the Composite Model and Aggregations properties. + +When running this utility against an Azure Analysis Services model, it will upgrade the model to Compatibility Level 1465, this change is not reversible via this utility. The utility will only update the metadata of tables listed in the configuration file, but it will both add and remove Aggregation column definitions, so that the model metadata will match the configuration file. Partition mode changes will be applied to all the partitions defined on the table. + +## Example Usage +SetUpAggs.exe apply -Server asazure://server/instance -ConfigFile AggsConfig.json + +## Steps +* Parse command line arguments, show basic help +* Connect to model +* Validate configuration file against the model, exit with relevant error message if issue is found in configuration or model +* Update model compatibility level if current compatibility level is below 1465 +* Apply configuration changes to model +** For each table +*** For each partition, update partition mode to Dual, DirectQuery, Import, or Default as specified in config +*** For each column, remove any AlternateOf definitions that do not match an aggregation rule in the config +*** For each rule, add the AlternateOf definition to the model if it does not already exist +* If changes were made to the model, run the ExpandFull on the model +* If changes were made to the model, refresh the tables that were modified + +## Configuration File Example + +{ + "database": { + "name": "AdventureWorksAggsProvider", + "tables": [ + { + "name": "DimGeography", + "mode": "Dual" + }, + { + "name": "DimCustomer", + "mode": "Dual" + }, + { + "name": "DimDate", + "mode": "Dual" + }, + { + "name": "DimProductSubcategory", + "mode": "Dual" + }, + { + "name": "DimProductCategory", + "mode": "Dual" + }, + { + "name": "FactInternetSalesAgg", + "mode": "Import", + "aggregationRules": [ + { + "aggTableColumn": "OrderDateKey", + "summarization": "GroupBy", + "detailTable": "FactInternetSales", + "detailTableColumn": "OrderDateKey" + }, + { + "aggTableColumn": "CustomerKey", + "summarization": "GroupBy", + "detailTable": "FactInternetSales", + "detailTableColumn": "CustomerKey" + }, + { + "aggTableColumn": "ProductSubcategoryKey", + "summarization": "GroupBy", + "detailTable": "DimProduct", + "detailTableColumn": "ProductSubcategoryKey" + }, + { + "aggTableColumn": "SalesAmount_Sum", + "summarization": "Sum", + "detailTable": "FactInternetSales", + "detailTableColumn": "SalesAmount" + }, + { + "aggTableColumn": "UnitPrice_Sum", + "summarization": "Sum", + "detailTable": "FactInternetSales", + "detailTableColumn": "UnitPrice" + }, + { + "aggTableColumn": "FactInternetSales_Count", + "summarization": "CountTableRows", + "detailTable": "FactInternetSales" + } + ] + } + ] + } +} + + + + diff --git a/SetUpAggs/SetUpAggs.csproj b/SetUpAggs/SetUpAggs.csproj index d6819e0..e0f1aab 100644 --- a/SetUpAggs/SetUpAggs.csproj +++ b/SetUpAggs/SetUpAggs.csproj @@ -11,6 +11,21 @@ v4.6.1 512 true + publish\ + true + Disk + false + Foreground + 7 + Days + false + false + true + 0 + 1.0.0.%2a + false + false + true AnyCPU @@ -67,8 +82,25 @@ + + + + + + + + + False + Microsoft .NET Framework 4.6.1 %28x86 and x64%29 + true + + + False + .NET Framework 3.5 SP1 + false + \ No newline at end of file