release 0.0.3\nfull_sorting_merge for dw.fact_stock_minute

models/dw/dw.fact_stock_minute.sql

@@ -5,7 +5,11 @@
         order_by='ts_code, time_id',
         unique_key=['ts_code', 'time_id'],
         partition_by=['cast(floor(time_id / 1000000) As Int32)'],
-        query_settings={"join_algorithm": "'partial_merge'"}
+        query_settings={
+            "join_algorithm": "'full_sorting_merge'", 
+            "max_rows_in_set_to_optimize_join": "0",
+            "max_bytes_before_external_sort": "'1000M'"
+        }
     )
 }}
 

models/dw/schema.yml

@@ -2,291 +2,289 @@
 version: 2
 
 models:
-  - name: dw.dim_exchange
+- name: dw.dim_exchange
-    description: "交易所"
+  description: "交易所"
-    columns:
+  columns:
-      - name: exchange_id
+  - name: exchange_id
-        description: "The primary key for this table"
+    description: "The primary key for this table"
-        tests:
+    tests:
-          - unique
+    - unique
-          - not_null
+    - not_null
-      - name: ts_exchange_code
+  - name: ts_exchange_code
-        description: "交易所代码"
+    description: "交易所代码"
-        tests:
+    tests:
-          - not_null
+    - not_null
-          - unique
+    - unique
-      - name: exchange_name
+  - name: exchange_name
-        description: "交易所名称"
+    description: "交易所名称"
-        tests:
+    tests:
-          - not_null
+    - not_null
-          - unique
+    - unique
-  - name: dw.dim_date
+- name: dw.dim_date
+  description: "日期"
+  tests:
+  - dbt_utils.unique_combination_of_columns:
+      combination_of_columns:
+      - year
+      - month
+      - day
+  columns:
+  - name: date_id
+    description: "日期键"
+    tests:
+    - unique
+    - not_null
+  - name: full_date
     description: "日期"
     tests:
-      - dbt_utils.unique_combination_of_columns:
+    - not_null
-          combination_of_columns: 
+    - unique
-            - year
+  - name: year
-            - month
+    description: "年"
-            - day
+    tests:
-    columns:
+    - not_null
-      - name: date_id
+  - name: quarter
-        description: "日期键"
+    description: "季度"
-        tests:
+    tests:
-          - unique
+    - not_null
-          - not_null
+  - name: month
-      - name: full_date
+    description: "月"
-        description: "日期"
+    tests:
-        tests:
+    - not_null
-          - not_null
+  - name: day
-          - unique
+    description: "日"
-      - name: year
+    tests:
-        description: "年"
+    - not_null
-        tests:
+  - name: week_num_of_year
-          - not_null
+    description: "年中第几周"
-      - name: quarter
+    tests:
-        description: "季度"
+    - not_null
-        tests:
+  - name: day_num_of_week
-          - not_null
+    description: "星期"
-      - name: month
+    tests:
-        description: "月"
+    - not_null
-        tests:
+- name: dw.dim_time
-          - not_null
+  description: "时间"
-      - name: day
+  tests:
-        description: "日"
+  - dbt_utils.unique_combination_of_columns:
-        tests:
+      combination_of_columns:
-          - not_null
+      - date_id
-      - name: week_num_of_year
+      - hour
-        description: "年中第几周"
+      - minute
-        tests:
+  columns:
-          - not_null
+  - name: time_id
-      - name: day_num_of_week
+    description: "时间键"
-        description: "星期"
+    tests:
-        tests:
+    - unique
-          - not_null
+    - not_null
-  - name: dw.dim_time
+  - name: full_time
     description: "时间"
     tests:
-      - dbt_utils.unique_combination_of_columns:
+    - not_null
-          combination_of_columns: 
+    - unique
-            - date_id
+  - name: date_id
-            - hour
+    description: "日期键"
-            - minute
-    columns:
-      - name: time_id
-        description: "时间键"
-        tests:
-          - unique
-          - not_null
-      - name: full_time
-        description: "时间"
-        tests:
-          - not_null
-          - unique
-      - name: date_id
-        description: "日期键"
-        tests:
-          - not_null
-      - name: hour
-        description: "小时"
-        tests:
-          - not_null
-      - name: minute
-        description: "分钟"
-        tests:
-          - not_null
-  - name: dw.dim_hs_calendar
-    description: "沪深交易日历"
     tests:
-      - dbt_utils.unique_combination_of_columns:
+    - not_null
-          combination_of_columns:
+  - name: hour
-            - exchange_id
+    description: "小时"
-            - date_id
-    columns:
-      - name: exchange_id
-        description: "The primary key for this table"
-        tests:
-          - not_null
-      - name: date_id
-        description: "交易日期键"
-        tests:
-          - not_null
-      - name: is_open
-        description: "是否交易"
-        tests:
-          - not_null
-  - name: dw.dim_stock
-    description: "股票"
     tests:
-      - dbt_utils.unique_combination_of_columns:
+    - not_null
-          combination_of_columns:
+  - name: minute
-            - exchange_id
+    description: "分钟"
-            - code
-    columns:
-      - name: ts_code
-        description: "tushare 股票编码"
-        tests:
-          - unique
-          - not_null
-      - name: exchange_id
-        description: "交易所键"
-        tests:
-          - not_null
-      - name: code
-        description: "股票代码"
-        tests:
-          - not_null
-      - name: name
-        description: "股票名称"
-        tests:
-          - not_null
-          - unique
-      - name: full_name
-        description: "股票全称"
-        tests:
-          - not_null
-          - unique
-      - name: name_en
-        description: "股票英文名称"
-        tests:
-          - not_null
-      - name: cn_spell
-        description: "拼音缩写"
-        tests:
-          - not_null
-      - name: area
-        description: "地区"
-      - name: industry
-        description: "行业"
-      - name: market
-        description: "市场类型 （主板/中小板/创业板）"
-      - name: currency
-        description: "交易货币"
-      - name: list_date
-        description: "上市日期"
-      - name: delist_date
-        description: "退市日期"
-      - name: is_active
-        description: "当前是否在市"
-        tests:
-          - not_null
-      - name: is_hongkong_connect
-        description: "是否沪深通标的"
-        tests:
-          - not_null
-      - name: actual_controller
-        description: "实际控制人"
-      - name: actual_controller_enterprise_type
-        description: "实际控制人企业类型"
-      - name: dt
-        description: "更新时间"
-  - name: dw.fact_stock_daily
-    description: "股票日数据"
     tests:
-      - dbt_utils.unique_combination_of_columns:
+    - not_null
-          combination_of_columns:
+- name: dw.dim_hs_calendar
-            - ts_code
+  description: "沪深交易日历"
-            - date_id
+  tests:
-    columns:
+  - dbt_utils.unique_combination_of_columns:
-      - name: ts_code
+      combination_of_columns:
-        description: "股票键"
+      - exchange_id
-        tests:
+      - date_id
-          - not_null
+  columns:
-      - name: date_id
+  - name: exchange_id
-        description: "日期键"
+    description: "The primary key for this table"
-        tests:
-          - not_null
-      - name: open
-        description: "开盘价"
-      - name: high
-        description: "最高价"
-      - name: low
-        description: "最低价"
-      - name: close
-        description: "收盘价"
-      - name: pre_close
-        description: "昨收价"
-      - name: change
-        description: "涨跌额"
-      - name: pct_chg
-        description: "涨跌幅"
-      - name: vol
-        description: "成交量"
-      - name: amount
-        description: "成交额"
-      - name: adj_factor
-        description: "复权因子"
-      - name: turnover_rate
-        description: "换手率"
-      - name: turnover_rate_free
-        description: "换手率（自由流通股）"
-      - name: volume_ratio
-        description: "量比"
-      - name: pe
-        description: "市盈率（总市值/净利润， 亏损的PE为空）"
-      - name: pe_ttm
-        description: "市盈率（TTM）"
-      - name: pb
-        description: "市净率（总市值/净资产）"
-      - name: ps
-        description: "市销率"
-      - name: ps_ttm
-        description: "市销率（TTM）"
-      - name: dv_ratio
-        description: "股息率"
-      - name: dv_ttm
-        description: "股息率（TTM）"
-      - name: total_share
-        description: "总股本（万股）"
-      - name: float_share
-        description: "流通股本（万股）"
-      - name: free_share
-        description: "自由流通股本（万）"
-      - name: total_mv
-        description: "总市值（万元）"
-      - name: circ_mv
-        description: "流通市值（万元）"
-  - name: dw.fact_stock_minute
-    description: "股票分钟数据"
     tests:
-      - dbt_utils.unique_combination_of_columns:
+    - not_null
-          combination_of_columns:
+  - name: date_id
-            - ts_code
+    description: "交易日期键"
-            - time_id
+    tests:
-    columns:
+    - not_null
-      - name: ts_code
+  - name: is_open
-        description: "股票键"
+    description: "是否交易"
-        data_type: "String"
+    tests:
-        tests:
+    - not_null
-          - not_null
+- name: dw.dim_stock
-      - name: time_id
+  description: "股票"
-        description: "时间键"
+  tests:
-        data_type: "Int64"
+  - dbt_utils.unique_combination_of_columns:
-        tests:
+      combination_of_columns:
-          - not_null
+      - exchange_id
-      - name: open
+      - code
-        description: "开盘价"
+  columns:
-        data_type: "Nullable(Decimal(16,2))"
+  - name: ts_code
-      - name: high
+    description: "tushare 股票编码"
-        description: "最高价"
+    tests:
-        data_type: "Nullable(Decimal(16,2))"
+    - unique
-      - name: low
+    - not_null
-        description: "最低价"
+  - name: exchange_id
-        data_type: "Nullable(Decimal(16,2))"
+    description: "交易所键"
-      - name: close
+    tests:
-        description: "收盘价"
+    - not_null
-        data_type: "Nullable(Decimal(16,2))"
+  - name: code
-      - name: vol
+    description: "股票代码"
-        description: "成交量"
+    tests:
-        data_type: "Nullable(Int32)"
+    - not_null
-      - name: amount
+  - name: name
-        description: "成交额"
+    description: "股票名称"
-        data_type: "Nullable(Decimal(18,2))"
+    tests:
-      - name: adj_factor
+    - not_null
-        description: "复权因子"
+    - unique
-        data_type: "Decimal(16,6)"
+  - name: full_name
-        tests:
+    description: "股票全称"
-          - not_null
+    tests:
-      - name: dt
+    - not_null
-        description: "时间"
+    - unique
-        data_type: "DateTime"
+  - name: name_en
-        
+    description: "股票英文名称"
+    tests:
+    - not_null
+  - name: cn_spell
+    description: "拼音缩写"
+    tests:
+    - not_null
+  - name: area
+    description: "地区"
+  - name: industry
+    description: "行业"
+  - name: market
+    description: "市场类型 （主板/中小板/创业板）"
+  - name: currency
+    description: "交易货币"
+  - name: list_date
+    description: "上市日期"
+  - name: delist_date
+    description: "退市日期"
+  - name: is_active
+    description: "当前是否在市"
+    tests:
+    - not_null
+  - name: is_hongkong_connect
+    description: "是否沪深通标的"
+    tests:
+    - not_null
+  - name: actual_controller
+    description: "实际控制人"
+  - name: actual_controller_enterprise_type
+    description: "实际控制人企业类型"
+  - name: dt
+    description: "更新时间"
+- name: dw.fact_stock_daily
+  description: "股票日数据"
+  tests:
+  - dbt_utils.unique_combination_of_columns:
+      combination_of_columns:
+      - ts_code
+      - date_id
+  columns:
+  - name: ts_code
+    description: "股票键"
+    tests:
+    - not_null
+  - name: date_id
+    description: "日期键"
+    tests:
+    - not_null
+  - name: open
+    description: "开盘价"
+  - name: high
+    description: "最高价"
+  - name: low
+    description: "最低价"
+  - name: close
+    description: "收盘价"
+  - name: pre_close
+    description: "昨收价"
+  - name: change
+    description: "涨跌额"
+  - name: pct_chg
+    description: "涨跌幅"
+  - name: vol
+    description: "成交量"
+  - name: amount
+    description: "成交额"
+  - name: adj_factor
+    description: "复权因子"
+  - name: turnover_rate
+    description: "换手率"
+  - name: turnover_rate_free
+    description: "换手率（自由流通股）"
+  - name: volume_ratio
+    description: "量比"
+  - name: pe
+    description: "市盈率（总市值/净利润， 亏损的PE为空）"
+  - name: pe_ttm
+    description: "市盈率（TTM）"
+  - name: pb
+    description: "市净率（总市值/净资产）"
+  - name: ps
+    description: "市销率"
+  - name: ps_ttm
+    description: "市销率（TTM）"
+  - name: dv_ratio
+    description: "股息率"
+  - name: dv_ttm
+    description: "股息率（TTM）"
+  - name: total_share
+    description: "总股本（万股）"
+  - name: float_share
+    description: "流通股本（万股）"
+  - name: free_share
+    description: "自由流通股本（万）"
+  - name: total_mv
+    description: "总市值（万元）"
+  - name: circ_mv
+    description: "流通市值（万元）"
+- name: dw.fact_stock_minute
+  description: "股票分钟数据"
+  tests:
+  - dbt_utils.unique_combination_of_columns:
+      combination_of_columns:
+      - ts_code
+      - time_id
+  columns:
+  - name: ts_code
+    description: "股票键"
+    data_type: "String"
+    tests:
+    - not_null
+  - name: time_id
+    description: "时间键"
+    data_type: "Int64"
+    tests:
+    - not_null
+  - name: open
+    description: "开盘价"
+    data_type: "Nullable(Decimal(16,2))"
+  - name: high
+    description: "最高价"
+    data_type: "Nullable(Decimal(16,2))"
+  - name: low
+    description: "最低价"
+    data_type: "Nullable(Decimal(16,2))"
+  - name: close
+    description: "收盘价"
+    data_type: "Nullable(Decimal(16,2))"
+  - name: vol
+    description: "成交量"
+    data_type: "Nullable(Int32)"
+  - name: amount
+    description: "成交额"
+    data_type: "Nullable(Decimal(18,2))"
+  - name: adj_factor
+    description: "复权因子"
+    data_type: "Decimal(16,6)"
+  - name: dt
+    description: "时间"
+    data_type: "DateTime"
+