release: 0.1.0 add ods for minutes trade data

models/dw/dw.dim_time.sql

@@ -2,14 +2,16 @@
     config(
         materialized='table',
         engine='MergeTree',
-        order_by='time_id'
+        order_by='(date_id_str, full_time_str, time_id)'
     ) 
 }}
-Select cast(formatDateTime(`full_time`, '%Y%m%d%H%i') As Int64) As time_id
+Select toYYYYMMDDhhmmss(`full_time`) As time_id
 	, `full_time`
 	, `date_id`
 	, hour(`full_time`) As `hour`
 	, minute(`full_time`) As `minute`
+	, formatDateTimeInJodaSyntax(`full_time`, 'yyyy-MM-dd HH:mm:ss') As full_time_str
+	, Cast(`date_id` As String) As date_id_str
 From(
 	SELECT 
 		arrayJoin(

models/dw/dw.fact_stock_daily.sql

@@ -5,6 +5,10 @@
         order_by='ts_code, date_id',
         unique_key=['ts_code', 'date_id'],
         incremental_strategy='delete+insert',
+        query_settings={
+            "join_algorithm": "'full_sorting_merge'", 
+            "max_bytes_before_external_sort": "'1000M'"
+        }
     )
 }}
 Select 

models/dw/dw.fact_stock_minute.sql

@@ -1,38 +1,42 @@
 {{ 
     config(
-        materialized='materialized_view',
+        materialized='incremental',
         engine='MergeTree',
         order_by='ts_code, time_id',
         unique_key=['ts_code', 'time_id'],
+        incremental_strategy='delete+insert',
         partition_by=['cast(floor(time_id / 1000000) As Int32)'],
         query_settings={
             "join_algorithm": "'full_sorting_merge'", 
-            "max_rows_in_set_to_optimize_join": "0",
+            "max_bytes_before_external_sort": "'1000M'",
-            "max_bytes_before_external_sort": "'1000M'"
+            "max_bytes_before_external_group_by":"'1000M'",
         }
     )
 }}
 
 Select 
-    StgMinutes.ts_code As ts_code,
+    OdsMinutes.ts_code As ts_code,
-    DimTime.time_id As time_id,
+    OdsMinutes.time_id As time_id,
-    StgMinutes.close As close,
+    OdsMinutes.close As close,
-    StgMinutes.open As open,
+    OdsMinutes.open As open,
-    StgMinutes.high As high,
+    OdsMinutes.high As high,
-    StgMinutes.low As low,
+    OdsMinutes.low As low,
-    StgMinutes.vol As vol,
+    OdsMinutes.vol As vol,
-    StgMinutes.amount As amount,
+    OdsMinutes.amount As amount,
-    StgAdjFactor.adj_factor As adj_factor,
+    OdsAdjFactor.adj_factor As adj_factor,
     now() As dt
-From {{ source('finance', 'stg.tushare_minutes') }} As StgMinutes 
+From {{ ref('ods.tushare_minutes') }} As OdsMinutes
-Inner Join {{ ref('dw.dim_time') }} As DimTime
+Any Left Join {{ ref('dw.dim_time') }} As DimTime
-    On StgMinutes.trade_time = formatDateTimeInJodaSyntax(full_time, 'yyyy-MM-dd HH:mm:ss')
+    On OdsMinutes.time_id = DimTime.time_id
-Any Left Join {{ source('finance', 'stg.tushare_adj_factor') }} As StgAdjFactor
+Any Left Join {{ ref('ods.tushare_adj_factor') }} As OdsAdjFactor
-    On StgMinutes.ts_code = StgAdjFactor.ts_code
+    On OdsMinutes.ts_code = OdsAdjFactor.ts_code
-        And StgAdjFactor.trade_date = Cast(DimTime.date_id As String)
+        And OdsAdjFactor.date_id = DimTime.date_id
 where 1 = 1
-{% if 'dev' in target.name %}
+{% if is_incremental() %}
-And StgMinutes.trade_time >= formatDateTimeInJodaSyntax(today(), 'yyyy-01-01 00:00:00')
+    And OdsMinutes.time_id >= (Select max(`time_id`) From {{ this }})
-{% elif target.name == 'test' %}
+{% endif %}
-And StgMinutes.trade_time >= formatDateTimeInJodaSyntax(dateAdd(Year,-1,today()), 'yyyy-01-01 00:00:00')
+{% if 'dev' in target.name %}
+    And OdsMinutes.time_id >= toYYYYMMDDhhmmss(dateAdd(Month,-1,today()))
+{% elif target.name == 'test' %}
+    And OdsMinutes.time_id >= toYYYYMMDDhhmmss(dateAdd(Year,-1,today()))
 {% endif %}

models/dw/schema.yml

@@ -94,6 +94,15 @@ models:
     description: "分钟"
     tests:
     - not_null
+  - name: full_time_str
+    description: "时间字符串（yyyy-MM-dd HH:mm:ss）"
+    tests:
+    - not_null
+    - unique
+  - name: date_id_str
+    description: "日期字符串（yyyy-MM-dd）"
+    tests:
+    - not_null
 - name: dw.dim_hs_calendar
   description: "沪深交易日历"
   tests:

models/finance_source.yml

@@ -4,250 +4,250 @@ sources:
   schema: '{{ env_var("FINANCE_CLICKHOUSE_DATABASE") }}'
   description: "The source for financial data"
   tables:
-    - name: stg.tushare_trade_calendar
+  - name: stg.tushare_trade_calendar
-      description: >
+    description: >
-        The source table for the trade calendar data from Tushare
+      The source table for the trade calendar data from Tushare
+    tests:
+    - dbt_utils.unique_combination_of_columns:
+        combination_of_columns:
+        - exchange
+        - cal_date
+    columns:
+    - name: exchange
+      description: "The exchange code"
       tests:
-      - dbt_utils.unique_combination_of_columns:
+      - not_null
-          combination_of_columns:
+    - name: cal_date
-            - exchange
+      description: "The date"
-            - cal_date
-      columns:
-      - name: exchange
-        description: "The exchange code"
-        tests:
-          - not_null
-      - name: cal_date
-        description: "The date"
-        tests:
-          - not_null
-      - name: is_open
-        description: "Whether the exchange is open"
-        tests:
-          - not_null
-      - name: pretrade_date
-        description: "The previous trading date"
-    - name: stg.tushare_stock_basic
-      columns:
-        - name: ts_code
-          description: TS代码
-          tests:
-            - not_null
-        - name: symbol
-          description: 股票代码
-        - name: name
-          description: 股票名称
-        - name: area
-          description: 所在地域
-        - name: industry
-          description: 所属行业
-        - name: fullname
-          description: 股票全称
-        - name: enname
-          description: 英文全称
-        - name: cnspell
-          description: 拼音缩写
-        - name: market
-          description: 市场类型 （主板/中小板/创业板）
-        - name: exchange
-          description: 交易所代码
-          tests:
-            - not_null
-        - name: curr_type
-          description: 交易货币
-        - name: list_status
-          description: 上市状态： L上市 D退市 P暂停上市
-        - name: list_date
-          description: 上市日期
-        - name: delist_date
-          description: 退市日期
-        - name: is_hs
-          description: 是否沪深港通标的，N否 H沪股通 S深股通
-        - name: act_name
-          description: 实控人名称
-        - name: act_ent_type
-          description: 实控人企业性质
-        - name: dt
-          description: 最后修改时间
-      unique_composite:
-        - name: unique_exchange_ts_code
-          columns:
-            - exchange
-            - ts_code
-    - name: stg.tushare_daily
-      description: "沪深股票交易日线数据"
       tests:
-        - dbt_utils.unique_combination_of_columns:
+      - not_null
-            combination_of_columns:
+    - name: is_open
-              - ts_code
+      description: "Whether the exchange is open"
-              - trade_date
-      columns:
-        - name: ts_code
-          description: TS代码
-          tests:
-            - not_null
-        - name: trade_date
-          description: 交易日期
-          tests:
-            - not_null
-        - name: open
-          description: 开盘价
-          tests:
-            - not_null
-        - name: high
-          description: 最高价
-          tests:
-            - not_null
-        - name: low
-          description: 最低价
-          tests:
-            - not_null
-        - name: close
-          description: 收盘价
-          tests:
-            - not_null
-        - name: pre_close
-          description: 昨收价
-          tests:
-            - not_null
-        - name: change
-          description: 涨跌额
-          tests:
-            - not_null
-        - name: pct_chg
-          description: 涨跌幅
-          tests:
-            - not_null
-        - name: vol
-          description: 成交量
-          tests:
-            - not_null
-        - name: amount
-          description: 成交额
-    - name: stg.tushare_daily_basic
-      description: "沪深股票每日指标数据"
       tests:
-        - dbt_utils.unique_combination_of_columns:
+      - not_null
-            combination_of_columns:
+    - name: pretrade_date
-              - ts_code
+      description: "The previous trading date"
-              - trade_date
+  - name: stg.tushare_stock_basic
+    columns:
+    - name: ts_code
+      description: TS代码
+      tests:
+      - not_null
+    - name: symbol
+      description: 股票代码
+    - name: name
+      description: 股票名称
+    - name: area
+      description: 所在地域
+    - name: industry
+      description: 所属行业
+    - name: fullname
+      description: 股票全称
+    - name: enname
+      description: 英文全称
+    - name: cnspell
+      description: 拼音缩写
+    - name: market
+      description: 市场类型 （主板/中小板/创业板）
+    - name: exchange
+      description: 交易所代码
+      tests:
+      - not_null
+    - name: curr_type
+      description: 交易货币
+    - name: list_status
+      description: 上市状态： L上市 D退市 P暂停上市
+    - name: list_date
+      description: 上市日期
+    - name: delist_date
+      description: 退市日期
+    - name: is_hs
+      description: 是否沪深港通标的，N否 H沪股通 S深股通
+    - name: act_name
+      description: 实控人名称
+    - name: act_ent_type
+      description: 实控人企业性质
+    - name: dt
+      description: 最后修改时间
+    unique_composite:
+    - name: unique_exchange_ts_code
+      columns:
+      - exchange
+      - ts_code
+  - name: stg.tushare_daily
+    description: "沪深股票交易日线数据"
+    tests:
+    - dbt_utils.unique_combination_of_columns:
+        combination_of_columns:
+        - ts_code
+        - trade_date
+    columns:
+    - name: ts_code
+      description: TS代码
+      tests:
+      - not_null
+    - name: trade_date
+      description: 交易日期
+      tests:
+      - not_null
+    - name: open
+      description: 开盘价
+      tests:
+      - not_null
+    - name: high
+      description: 最高价
+      tests:
+      - not_null
+    - name: low
+      description: 最低价
+      tests:
+      - not_null
+    - name: close
+      description: 收盘价
+      tests:
+      - not_null
+    - name: pre_close
+      description: 昨收价
+      tests:
+      - not_null
+    - name: change
+      description: 涨跌额
+      tests:
+      - not_null
+    - name: pct_chg
+      description: 涨跌幅
+      tests:
+      - not_null
+    - name: vol
+      description: 成交量
+      tests:
+      - not_null
+    - name: amount
+      description: 成交额
+  - name: stg.tushare_daily_basic
+    description: "沪深股票每日指标数据"
+    tests:
+    - dbt_utils.unique_combination_of_columns:
+        combination_of_columns:
+        - ts_code
+        - trade_date
 
-      columns:
+    columns:
-        - name: ts_code
+    - name: ts_code
-          description: "股票代码"
+      description: "股票代码"
-          data_type: String
+      data_type: String
-        - name: trade_date
+    - name: trade_date
-          description: "交易日期"
+      description: "交易日期"
-          data_type: String
+      data_type: String
-        - name: close
+    - name: close
-          description: "当日收盘价"
+      description: "当日收盘价"
-          data_type: Nullable(Float32)
+      data_type: Nullable(Float32)
-        - name: turnover_rate
+    - name: turnover_rate
-          description: "换手率（%）"
+      description: "换手率（%）"
-          data_type: Nullable(Float32)
+      data_type: Nullable(Float32)
-        - name: turnover_rate_f
+    - name: turnover_rate_f
-          description: "换手率（自由流通股）"
+      description: "换手率（自由流通股）"
-          data_type: Nullable(Float32)
+      data_type: Nullable(Float32)
-        - name: volume_ratio
+    - name: volume_ratio
-          description: "量比"
+      description: "量比"
-          data_type: Nullable(Float32)
+      data_type: Nullable(Float32)
-        - name: pe
+    - name: pe
-          description: "市盈率（总市值/净利润， 亏损的PE为空）"
+      description: "市盈率（总市值/净利润， 亏损的PE为空）"
-          data_type: Nullable(Float32)
+      data_type: Nullable(Float32)
-        - name: pe_ttm
+    - name: pe_ttm
-          description: "市盈率（TTM，亏损的PE为空）"
+      description: "市盈率（TTM，亏损的PE为空）"
-          data_type: Nullable(Float32)
+      data_type: Nullable(Float32)
-        - name: pb
+    - name: pb
-          description: "市净率（总市值/净资产）"
+      description: "市净率（总市值/净资产）"
-          data_type: Nullable(Float32)
+      data_type: Nullable(Float32)
-        - name: ps
+    - name: ps
-          description: "市销率"
+      description: "市销率"
-          data_type: Nullable(Float32)
+      data_type: Nullable(Float32)
-        - name: ps_ttm
+    - name: ps_ttm
-          description: "市销率（TTM）"
+      description: "市销率（TTM）"
-          data_type: Nullable(Float32)
+      data_type: Nullable(Float32)
-        - name: dv_ratio
+    - name: dv_ratio
-          description: "股息率 （%）"
+      description: "股息率 （%）"
-          data_type: Nullable(Float32)
+      data_type: Nullable(Float32)
-        - name: dv_ttm
+    - name: dv_ttm
-          description: "股息率（TTM）（%）"
+      description: "股息率（TTM）（%）"
-          data_type: Nullable(Float32)
+      data_type: Nullable(Float32)
-        - name: total_share
+    - name: total_share
-          description: "总股本 （万股）"
+      description: "总股本 （万股）"
-          data_type: Nullable(Float32)
+      data_type: Nullable(Float32)
-        - name: float_share
+    - name: float_share
-          description: "流通股本 （万股）"
+      description: "流通股本 （万股）"
-          data_type: Nullable(Float32)
+      data_type: Nullable(Float32)
-        - name: free_share
+    - name: free_share
-          description: "自由流通股本 （万）"
+      description: "自由流通股本 （万）"
-          data_type: Nullable(Float32)
+      data_type: Nullable(Float32)
-        - name: total_mv
+    - name: total_mv
-          description: "总市值 （万元）"
+      description: "总市值 （万元）"
-          data_type: Nullable(Float32)
+      data_type: Nullable(Float32)
-        - name: circ_mv
+    - name: circ_mv
-          description: "流通市值（万元）"
+      description: "流通市值（万元）"
-          data_type: Nullable(Float32)
+      data_type: Nullable(Float32)
-    - name: stg.tushare_adj_factor
+  - name: stg.tushare_adj_factor
-      description: "沪深股票复权因子"
+    description: "沪深股票复权因子"
+    tests:
+    - dbt_utils.unique_combination_of_columns:
+        combination_of_columns:
+        - ts_code
+        - trade_date
+    columns:
+    - name: ts_code
+      description: "股票代码"
+      data_type: String
+    - name: trade_date
+      description: "交易日期"
+      data_type: String
+    - name: adj_factor
+      description: "复权因子"
+      data_type: Nullable(Decimal(16, 6))
+    - name: dt
+      description: "更新时间"
+      data_type: DateTime
+  - name: stg.tushare_minutes
+    description: "沪深分钟级交易数据"
+    tests:
+    - dbt_utils.unique_combination_of_columns:
+        combination_of_columns:
+        - ts_code
+        - trade_time
+        config:
+          where: "left(trade_time,7)>=concat(cast(year(today())-1 As String), '-01')"
+    columns:
+    - name: ts_code
+      description: "The stock code."
+      data_type: String
       tests:
-        - dbt_utils.unique_combination_of_columns:
+      - not_null
-            combination_of_columns:
+    - name: trade_time
-              - ts_code
+      description: "The trading time."
-              - trade_date
+      data_type: String
-      columns:
-        - name: ts_code
-          description: "股票代码"
-          data_type: String
-        - name: trade_date
-          description: "交易日期"
-          data_type: String
-        - name: adj_factor
-          description: "复权因子"
-          data_type: Nullable(Decimal(16, 6))
-        - name: dt
-          description: "更新日期"
-          data_type: DateTime
-    - name: stg.tushare_minutes
-      description: "沪深分钟级交易数据"
       tests:
-        - dbt_utils.unique_combination_of_columns:
+      - not_null
-            combination_of_columns:
+    - name: close
-              - ts_code
+      description: "The closing price."
-              - trade_time
+      data_type: Nullable(Float32)
-            config:
+    - name: open
-              where: "left(trade_time,7)>=concat(cast(year(today())-1 As String), '-01')"
+      description: "The opening price."
-      columns:
+      data_type: Nullable(Float32)
-        - name: ts_code
+    - name: high
-          description: "The stock code."
+      description: "The highest price."
-          data_type: String
+      data_type: Nullable(Float32)
-          tests:
+    - name: low
-            - not_null
+      description: "The lowest price."
-        - name: trade_time
+      data_type: Nullable(Float32)
-          description: "The trading time."
+    - name: vol
-          data_type: String
+      description: "The volume of trades."
-          tests:
+      data_type: Nullable(Float32)
-            - not_null
+    - name: amount
-        - name: close
+      description: "The amount of trades."
-          description: "The closing price."
+      data_type: Nullable(Float32)
-          data_type: Nullable(Float32)
+    meta:
-        - name: open
+      engine: ReplacingMergeTree
-          description: "The opening price."
+      partition_by: "left(trade_time, 7)"
-          data_type: Nullable(Float32)
+      order_by: "(ts_code, trade_time)"
-        - name: high
+      settings:
-          description: "The highest price."
+        index_granularity: 8192
-          data_type: Nullable(Float32)
-        - name: low
-          description: "The lowest price."
-          data_type: Nullable(Float32)
-        - name: vol
-          description: "The volume of trades."
-          data_type: Nullable(Float32)
-        - name: amount
-          description: "The amount of trades."
-          data_type: Nullable(Float32)
-      meta:
-        engine: ReplacingMergeTree
-        partition_by: "left(trade_time, 7)"
-        order_by: "(ts_code, trade_time)"
-        settings:
-          index_granularity: 8192

models/ods/ods.tushare_adj_factor.sql

@@ -0,0 +1,19 @@
+{{ 
+    config(
+        materialized='incremental',
+        engine="MergeTree",
+        order_by="ts_code, date_id",
+        unique_key=['ts_code', 'date_id'],
+        incremental_strategy='delete+insert',
+    )
+}}
+
+Select
+    ts_code As ts_code,
+    toYYYYMMDD(toDate(trade_date)) As date_id,
+    adj_factor As adj_factor,
+    dt As dt
+From {{ source('finance', 'stg.tushare_adj_factor') }} As StgAdjFactor
+{% if is_incremental() %}
+Where `trade_date` >= (Select formatDateTimeInJodaSyntax(YYYYMMDDToDate(max(`date_id`)), 'yyyy-MM-dd') From {{ this }})
+{% endif %}

models/ods/ods.tushare_minutes.sql

@@ -0,0 +1,29 @@
+{{ 
+    config(
+        materialized='incremental',
+        engine="MergeTree",
+        order_by='ts_code, time_id',
+        unique_key=['ts_code', 'time_id'],
+        incremental_strategy='delete+insert',
+    )
+}}
+
+Select 
+    StgMinutes.ts_code As ts_code,
+    toYYYYMMDDhhmmss(toDateTime(`StgMinutes`.`trade_time`)) As time_id,
+    StgMinutes.close As close,
+    StgMinutes.open As open,
+    StgMinutes.high As high,
+    StgMinutes.low As low,
+    StgMinutes.vol As vol,
+    StgMinutes.amount As amount
+From {{ source('finance', 'stg.tushare_minutes') }} As StgMinutes 
+where 1 = 1
+{% if is_incremental() %}
+    And StgMinutes.trade_time >= (Select formatDateTimeInJodaSyntax(YYYYMMDDhhmmssToDateTime(max(time_id)), 'yyyy-MM-dd hh:mm:ss') From {{ this }})
+{% endif %}
+{% if 'dev' in target.name %}
+    And StgMinutes.trade_time >= formatDateTimeInJodaSyntax(dateAdd(Month,-1,today()), 'yyyy-01-01 00:00:00')
+{% elif target.name == 'test' %}
+    And StgMinutes.trade_time >= formatDateTimeInJodaSyntax(dateAdd(Year,-1,today()), 'yyyy-01-01 00:00:00')
+{% endif %}

models/ods/schema.yml

@@ -0,0 +1,54 @@
+
+version: 2
+
+models:
+- name: ods.tushare_minutes
+  description: "沪深股票交易分钟线数据"
+  tests:
+  - dbt_utils.unique_combination_of_columns:
+      combination_of_columns:
+      - ts_code
+      - time_id
+  columns:
+  - name: ts_code
+    description: TS代码
+    tests:
+    - not_null
+  - name: time_id
+    description: 交易日期
+    tests:
+    - not_null
+  - name: close
+    description: 收盘价
+  - name: open
+    description: 开盘价
+  - name: high
+    description: 最高价
+  - name: low
+    description: 最低价
+  - name: vol
+    description: 成交量
+  - name: amount
+    description: 成交额
+  - name: dt
+    description: 更新时间
+- name: ods.tushare_adj_factor
+  description: "沪深股票复权因子"
+  tests:
+  - dbt_utils.unique_combination_of_columns:
+      combination_of_columns:
+      - ts_code
+      - date_id
+  columns:
+  - name: ts_code
+    description: TS代码
+    tests:
+    - not_null
+  - name: date_id
+    description: 交易日期
+    tests:
+    - not_null
+  - name: adj_factor
+    description: 复权因子
+  - name: dt
+    description: 更新时间