Dockerfile

@@ -4,7 +4,7 @@ WORKDIR /app
 ### Pip
 USER root
 COPY ./requirements.txt /app/
-RUN pip config set global.index-url https://nexus.mujiannan.com:5001/repository/pypiserver/simple && \
+RUN pip3 config set global.index-url https://nexus.mujiannan.com:5001/repository/pypiserver/simple && \
     pip install -r requirements.txt
 USER 1001
 ## Copy files

doc-server.sh

@@ -1,2 +1,2 @@
 dbt docs generate
-dbt docs serve --host=0.0.0.0 --port=8080
+dbt docs serve

models/dw/dw.dim_date.sql

@@ -10,8 +10,7 @@
     ) 
 }}
 SELECT
-    toYYYYMMDD(date) AS date_id,
+    toUInt32(year(date) * 10000 + month(date) * 100 + day(date)) AS date_id,
-    formatDateTimeInJodaSyntax(date, 'yyyyMMdd') AS date_id_str,
     date AS full_date,
     toYear(date) AS year,
     toUInt8(quarter(date)) AS quarter,

models/dw/dw.dim_time.sql

@@ -2,16 +2,14 @@
     config(
         materialized='table',
         engine='MergeTree',
-        order_by='time_id',
+        order_by='time_id'
     ) 
 }}
-Select toYYYYMMDDhhmmss(`full_time`) As time_id
+Select cast(formatDateTime(`full_time`, '%Y%m%d%H%i') As Int64) As time_id
 	, `full_time`
 	, `date_id`
 	, hour(`full_time`) As `hour`
 	, minute(`full_time`) As `minute`
-	, formatDateTimeInJodaSyntax(`full_time`, 'yyyy-MM-dd HH:mm:ss') As full_time_str
-	, Cast(`date_id` As String) As date_id_str
 From(
 	SELECT 
 		arrayJoin(

models/dw/dw.fact_stock_daily.sql

@@ -2,12 +2,9 @@
     config(
         materialized='incremental',
         engine='MergeTree',
-        order_by='(date_id, ts_code)',
+        order_by='ts_code, date_id',
-        unique_key=['date_id', 'ts_code'],
+        unique_key=['ts_code', 'date_id'],
         incremental_strategy='delete+insert',
-        query_settings={
-            "join_algorithm": "'full_sorting_merge'",
-        }
     )
 }}
 Select 
@@ -38,7 +35,7 @@ Select
     StgDailyBasic.free_share As free_share,
     StgDailyBasic.total_mv As total_mv,
     StgDailyBasic.circ_mv As circ_mv
-From {{ source('finance', 'stg.tushare_daily') }} As StgDaily
+From {{ source('finance', 'stg.tushare_daily') }} As StgDaily Final
 Inner Join {{ ref('dw.dim_date') }} As DimDate
     On StgDaily.trade_date = Cast(DimDate.date_id As String)
 Any Left Join {{ source('finance', 'stg.tushare_daily_basic') }} As StgDailyBasic

models/dw/dw.fact_stock_minute.sql

@@ -1,40 +1,34 @@
 {{ 
     config(
-        materialized='incremental',
+        materialized='materialized_view',
         engine='MergeTree',
-        order_by='(time_id, ts_code)',
+        order_by='ts_code, time_id',
-        unique_key=['time_id', 'ts_code'],
+        unique_key=['ts_code', 'time_id'],
-        incremental_strategy='delete+insert',
+        partition_by=['cast(floor(time_id / 1000000) As Int32)'],
-        partition_by=['toYYYYMM(toDateTime(time_id))'],
+        query_settings={"join_algorithm": "'partial_merge'"}
-        query_settings={
-            "join_algorithm": "'full_sorting_merge'", 
-        }
     )
 }}
 
 Select 
-    OdsMinutes.ts_code As ts_code,
+    StgMinutes.ts_code As ts_code,
-    OdsMinutes.time_id As time_id,
+    DimTime.time_id As time_id,
-    OdsMinutes.close As close,
+    StgMinutes.close As close,
-    OdsMinutes.open As open,
+    StgMinutes.open As open,
-    OdsMinutes.high As high,
+    StgMinutes.high As high,
-    OdsMinutes.low As low,
+    StgMinutes.low As low,
-    OdsMinutes.vol As vol,
+    StgMinutes.vol As vol,
-    OdsMinutes.amount As amount,
+    StgMinutes.amount As amount,
-    OdsAdjFactor.adj_factor As adj_factor,
+    StgAdjFactor.adj_factor As adj_factor,
     now() As dt
-From {{ ref('ods.tushare_minutes') }} As OdsMinutes
+From {{ source('finance', 'stg.tushare_minutes') }} As StgMinutes 
-Any Left Join {{ ref('dw.dim_time') }} As DimTime
+Inner Join {{ ref('dw.dim_time') }} As DimTime
-    On OdsMinutes.time_id = DimTime.time_id
+    On StgMinutes.trade_time = formatDateTimeInJodaSyntax(full_time, 'yyyy-MM-dd HH:mm:ss')
-Any Left Join {{ ref('ods.tushare_adj_factor') }} As OdsAdjFactor
+Any Left Join {{ source('finance', 'stg.tushare_adj_factor') }} As StgAdjFactor
-    On OdsMinutes.ts_code = OdsAdjFactor.ts_code
+    On StgMinutes.ts_code = StgAdjFactor.ts_code
-        And OdsAdjFactor.date_id = DimTime.date_id
+        And StgAdjFactor.trade_date = Cast(DimTime.date_id As String)
 where 1 = 1
-{% if is_incremental() %}
-    And OdsMinutes.time_id >= (Select toYYYYMMDDhhmmss(toStartOfDay(YYYYMMDDhhmmssToDateTime(max(`time_id`)))) From {{ this }})
-{% endif %}
 {% if 'dev' in target.name %}
-    And OdsMinutes.time_id >= toYYYYMMDDhhmmss(dateAdd(Month,-1,today()))
+And StgMinutes.trade_time >= formatDateTimeInJodaSyntax(today(), 'yyyy-01-01 00:00:00')
 {% elif target.name == 'test' %}
-    And OdsMinutes.time_id >= toYYYYMMDDhhmmss(dateAdd(Year,-1,today()))
+And StgMinutes.trade_time >= formatDateTimeInJodaSyntax(dateAdd(Year,-1,today()), 'yyyy-01-01 00:00:00')
 {% endif %}

models/dw/schema.yml

@@ -2,303 +2,291 @@
 version: 2
 
 models:
-- name: dw.dim_exchange
+  - name: dw.dim_exchange
-  description: "交易所"
+    description: "交易所"
-  columns:
+    columns:
-  - name: exchange_id
+      - name: exchange_id
-    description: "The primary key for this table"
+        description: "The primary key for this table"
-    data_tests:
+        tests:
-    - unique
+          - unique
-    - not_null
+          - not_null
-  - name: ts_exchange_code
+      - name: ts_exchange_code
-    description: "交易所代码"
+        description: "交易所代码"
-    data_tests:
+        tests:
-    - not_null
+          - not_null
-    - unique
+          - unique
-  - name: exchange_name
+      - name: exchange_name
-    description: "交易所名称"
+        description: "交易所名称"
-    data_tests:
+        tests:
-    - not_null
+          - not_null
-    - unique
+          - unique
-- name: dw.dim_date
+  - name: dw.dim_date
-  description: "日期"
-  data_tests:
-  - dbt_utils.unique_combination_of_columns:
-      combination_of_columns:
-      - year
-      - month
-      - day
-  columns:
-  - name: date_id
-    description: "日期键"
-    data_tests:
-    - unique
-    - not_null
-  - name: date_id_str
-    description: "日期字符串（yyyyMMdd）"
-    data_tests:
-    - not_null
-    - unique
-  - name: full_date
     description: "日期"
-    data_tests:
+    tests:
-    - not_null
+      - dbt_utils.unique_combination_of_columns:
-    - unique
+          combination_of_columns: 
-  - name: year
+            - year
-    description: "年"
+            - month
-    data_tests:
+            - day
-    - not_null
+    columns:
-  - name: quarter
+      - name: date_id
-    description: "季度"
+        description: "日期键"
-    data_tests:
+        tests:
-    - not_null
+          - unique
-  - name: month
+          - not_null
-    description: "月"
+      - name: full_date
-    data_tests:
+        description: "日期"
-    - not_null
+        tests:
-  - name: day
+          - not_null
-    description: "日"
+          - unique
-    data_tests:
+      - name: year
-    - not_null
+        description: "年"
-  - name: week_num_of_year
+        tests:
-    description: "年中第几周"
+          - not_null
-    data_tests:
+      - name: quarter
-    - not_null
+        description: "季度"
-  - name: day_num_of_week
+        tests:
-    description: "星期"
+          - not_null
-    data_tests:
+      - name: month
-    - not_null
+        description: "月"
-- name: dw.dim_time
+        tests:
-  description: "时间"
+          - not_null
-  data_tests:
+      - name: day
-  - dbt_utils.unique_combination_of_columns:
+        description: "日"
-      combination_of_columns:
+        tests:
-      - date_id
+          - not_null
-      - hour
+      - name: week_num_of_year
-      - minute
+        description: "年中第几周"
-  columns:
+        tests:
-  - name: time_id
+          - not_null
-    description: "时间键"
+      - name: day_num_of_week
-    data_tests:
+        description: "星期"
-    - unique
+        tests:
-    - not_null
+          - not_null
-  - name: full_time
+  - name: dw.dim_time
     description: "时间"
-    data_tests:
+    tests:
-    - not_null
+      - dbt_utils.unique_combination_of_columns:
-    - unique
+          combination_of_columns: 
-  - name: date_id
+            - date_id
-    description: "日期键"
+            - hour
-    data_tests:
+            - minute
-    - not_null
+    columns:
-  - name: hour
+      - name: time_id
-    description: "小时"
+        description: "时间键"
-    data_tests:
+        tests:
-    - not_null
+          - unique
-  - name: minute
+          - not_null
-    description: "分钟"
+      - name: full_time
-    data_tests:
+        description: "时间"
-    - not_null
+        tests:
-  - name: full_time_str
+          - not_null
-    description: "时间字符串（yyyy-MM-dd HH:mm:ss）"
+          - unique
-    data_tests:
+      - name: date_id
-    - not_null
+        description: "日期键"
-    - unique
+        tests:
-  - name: date_id_str
+          - not_null
-    description: "日期字符串（yyyyMMdd）"
+      - name: hour
-    data_tests:
+        description: "小时"
-    - not_null
+        tests:
-- name: dw.dim_hs_calendar
+          - not_null
-  description: "沪深交易日历"
+      - name: minute
-  data_tests:
+        description: "分钟"
-  - dbt_utils.unique_combination_of_columns:
+        tests:
-      combination_of_columns:
+          - not_null
-      - exchange_id
+  - name: dw.dim_hs_calendar
-      - date_id
+    description: "沪深交易日历"
-  columns:
+    tests:
-  - name: exchange_id
+      - dbt_utils.unique_combination_of_columns:
-    description: "The primary key for this table"
+          combination_of_columns:
-    data_tests:
+            - exchange_id
-    - not_null
+            - date_id
-  - name: date_id
+    columns:
-    description: "交易日期键"
+      - name: exchange_id
-    data_tests:
+        description: "The primary key for this table"
-    - not_null
+        tests:
-  - name: is_open
+          - not_null
-    description: "是否交易"
+      - name: date_id
-    data_tests:
+        description: "交易日期键"
-    - not_null
+        tests:
-- name: dw.dim_stock
+          - not_null
-  description: "股票"
+      - name: is_open
-  data_tests:
+        description: "是否交易"
-  - dbt_utils.unique_combination_of_columns:
+        tests:
-      combination_of_columns:
+          - not_null
-      - exchange_id
+  - name: dw.dim_stock
-      - code
+    description: "股票"
-  columns:
+    tests:
-  - name: ts_code
+      - dbt_utils.unique_combination_of_columns:
-    description: "tushare 股票编码"
+          combination_of_columns:
-    data_tests:
+            - exchange_id
-    - unique
+            - code
-    - not_null
+    columns:
-  - name: exchange_id
+      - name: ts_code
-    description: "交易所键"
+        description: "tushare 股票编码"
-    data_tests:
+        tests:
-    - not_null
+          - unique
-  - name: code
+          - not_null
-    description: "股票代码"
+      - name: exchange_id
-    data_tests:
+        description: "交易所键"
-    - not_null
+        tests:
-  - name: name
+          - not_null
-    description: "股票名称"
+      - name: code
-    data_tests:
+        description: "股票代码"
-    - not_null
+        tests:
-    - unique
+          - not_null
-  - name: full_name
+      - name: name
-    description: "股票全称"
+        description: "股票名称"
-    data_tests:
+        tests:
-    - not_null
+          - not_null
-    - unique
+          - unique
-  - name: name_en
+      - name: full_name
-    description: "股票英文名称"
+        description: "股票全称"
-    data_tests:
+        tests:
-    - not_null
+          - not_null
-  - name: cn_spell
+          - unique
-    description: "拼音缩写"
+      - name: name_en
-    data_tests:
+        description: "股票英文名称"
-    - not_null
+        tests:
-  - name: area
+          - not_null
-    description: "地区"
+      - name: cn_spell
-  - name: industry
+        description: "拼音缩写"
-    description: "行业"
+        tests:
-  - name: market
+          - not_null
-    description: "市场类型 （主板/中小板/创业板）"
+      - name: area
-  - name: currency
+        description: "地区"
-    description: "交易货币"
+      - name: industry
-  - name: list_date
+        description: "行业"
-    description: "上市日期"
+      - name: market
-  - name: delist_date
+        description: "市场类型 （主板/中小板/创业板）"
-    description: "退市日期"
+      - name: currency
-  - name: is_active
+        description: "交易货币"
-    description: "当前是否在市"
+      - name: list_date
-    data_tests:
+        description: "上市日期"
-    - not_null
+      - name: delist_date
-  - name: is_hongkong_connect
+        description: "退市日期"
-    description: "是否沪深通标的"
+      - name: is_active
-    data_tests:
+        description: "当前是否在市"
-    - not_null
+        tests:
-  - name: actual_controller
+          - not_null
-    description: "实际控制人"
+      - name: is_hongkong_connect
-  - name: actual_controller_enterprise_type
+        description: "是否沪深通标的"
-    description: "实际控制人企业类型"
+        tests:
-  - name: dt
+          - not_null
-    description: "更新时间"
+      - name: actual_controller
-- name: dw.fact_stock_daily
+        description: "实际控制人"
-  description: "股票日数据"
+      - name: actual_controller_enterprise_type
-  data_tests:
+        description: "实际控制人企业类型"
-  - dbt_utils.unique_combination_of_columns:
+      - name: dt
-      combination_of_columns:
+        description: "更新时间"
-      - ts_code
+  - name: dw.fact_stock_daily
-      - date_id
+    description: "股票日数据"
-  columns:
+    tests:
-  - name: ts_code
+      - dbt_utils.unique_combination_of_columns:
-    description: "股票键"
+          combination_of_columns:
-    data_tests:
+            - ts_code
-    - not_null
+            - date_id
-  - name: date_id
+    columns:
-    description: "日期键"
+      - name: ts_code
-    data_tests:
+        description: "股票键"
-    - not_null
+        tests:
-  - name: open
+          - not_null
-    description: "开盘价"
+      - name: date_id
-  - name: high
+        description: "日期键"
-    description: "最高价"
+        tests:
-  - name: low
+          - not_null
-    description: "最低价"
+      - name: open
-  - name: close
+        description: "开盘价"
-    description: "收盘价"
+      - name: high
-  - name: pre_close
+        description: "最高价"
-    description: "昨收价"
+      - name: low
-  - name: change
+        description: "最低价"
-    description: "涨跌额"
+      - name: close
-  - name: pct_chg
+        description: "收盘价"
-    description: "涨跌幅"
+      - name: pre_close
-  - name: vol
+        description: "昨收价"
-    description: "成交量"
+      - name: change
-  - name: amount
+        description: "涨跌额"
-    description: "成交额"
+      - name: pct_chg
-  - name: adj_factor
+        description: "涨跌幅"
-    description: "复权因子"
+      - name: vol
-  - name: turnover_rate
+        description: "成交量"
-    description: "换手率"
+      - name: amount
-  - name: turnover_rate_free
+        description: "成交额"
-    description: "换手率（自由流通股）"
+      - name: adj_factor
-  - name: volume_ratio
+        description: "复权因子"
-    description: "量比"
+      - name: turnover_rate
-  - name: pe
+        description: "换手率"
-    description: "市盈率（总市值/净利润， 亏损的PE为空）"
+      - name: turnover_rate_free
-  - name: pe_ttm
+        description: "换手率（自由流通股）"
-    description: "市盈率（TTM）"
+      - name: volume_ratio
-  - name: pb
+        description: "量比"
-    description: "市净率（总市值/净资产）"
+      - name: pe
-  - name: ps
+        description: "市盈率（总市值/净利润， 亏损的PE为空）"
-    description: "市销率"
+      - name: pe_ttm
-  - name: ps_ttm
+        description: "市盈率（TTM）"
-    description: "市销率（TTM）"
+      - name: pb
-  - name: dv_ratio
+        description: "市净率（总市值/净资产）"
-    description: "股息率"
+      - name: ps
-  - name: dv_ttm
+        description: "市销率"
-    description: "股息率（TTM）"
+      - name: ps_ttm
-  - name: total_share
+        description: "市销率（TTM）"
-    description: "总股本（万股）"
+      - name: dv_ratio
-  - name: float_share
+        description: "股息率"
-    description: "流通股本（万股）"
+      - name: dv_ttm
-  - name: free_share
+        description: "股息率（TTM）"
-    description: "自由流通股本（万）"
+      - name: total_share
-  - name: total_mv
+        description: "总股本（万股）"
-    description: "总市值（万元）"
+      - name: float_share
-  - name: circ_mv
+        description: "流通股本（万股）"
-    description: "流通市值（万元）"
+      - name: free_share
-- name: dw.fact_stock_minute
+        description: "自由流通股本（万）"
-  description: "股票分钟数据"
+      - name: total_mv
-  data_tests:
+        description: "总市值（万元）"
-  - dbt_utils.unique_combination_of_columns:
+      - name: circ_mv
-      combination_of_columns:
+        description: "流通市值（万元）"
-      - ts_code
+  - name: dw.fact_stock_minute
-      - time_id
+    description: "股票分钟数据"
-  columns:
+    tests:
-  - name: ts_code
+      - dbt_utils.unique_combination_of_columns:
-    description: "股票键"
+          combination_of_columns:
-    data_type: "String"
+            - ts_code
-    data_tests:
+            - time_id
-    - not_null
+    columns:
-  - name: time_id
+      - name: ts_code
-    description: "时间键"
+        description: "股票键"
-    data_type: "Int64"
+        data_type: "String"
-    data_tests:
+        tests:
-    - not_null
+          - not_null
-  - name: open
+      - name: time_id
-    description: "开盘价"
+        description: "时间键"
-    data_type: "Nullable(Decimal(16,2))"
+        data_type: "Int64"
-  - name: high
+        tests:
-    description: "最高价"
+          - not_null
-    data_type: "Nullable(Decimal(16,2))"
+      - name: open
-  - name: low
+        description: "开盘价"
-    description: "最低价"
+        data_type: "Nullable(Decimal(16,2))"
-    data_type: "Nullable(Decimal(16,2))"
+      - name: high
-  - name: close
+        description: "最高价"
-    description: "收盘价"
+        data_type: "Nullable(Decimal(16,2))"
-    data_type: "Nullable(Decimal(16,2))"
+      - name: low
-  - name: vol
+        description: "最低价"
-    description: "成交量"
+        data_type: "Nullable(Decimal(16,2))"
-    data_type: "Nullable(Int32)"
+      - name: close
-  - name: amount
+        description: "收盘价"
-    description: "成交额"
+        data_type: "Nullable(Decimal(16,2))"
-    data_type: "Nullable(Decimal(18,2))"
+      - name: vol
-  - name: adj_factor
+        description: "成交量"
-    description: "复权因子"
+        data_type: "Nullable(Int32)"
-    data_type: "Decimal(16,6)"
+      - name: amount
-  - name: dt
+        description: "成交额"
-    description: "时间"
+        data_type: "Nullable(Decimal(18,2))"
-    data_type: "DateTime"
+      - name: adj_factor
-
+        description: "复权因子"
+        data_type: "Decimal(16,6)"
+        tests:
+          - not_null
+      - name: dt
+        description: "时间"
+        data_type: "DateTime"
+        

models/finance_source.yml

@@ -4,250 +4,250 @@ sources:
   schema: '{{ env_var("FINANCE_CLICKHOUSE_DATABASE") }}'
   description: "The source for financial data"
   tables:
-  - name: stg.tushare_trade_calendar
+    - name: stg.tushare_trade_calendar
-    description: >
+      description: >
-      The source table for the trade calendar data from Tushare
+        The source table for the trade calendar data from Tushare
-    data_tests:
+      tests:
-    - dbt_utils.unique_combination_of_columns:
+      - dbt_utils.unique_combination_of_columns:
-        combination_of_columns:
+          combination_of_columns:
-        - exchange
+            - exchange
-        - cal_date
+            - cal_date
-    columns:
-    - name: exchange
-      description: "The exchange code"
-      data_tests:
-      - not_null
-    - name: cal_date
-      description: "The date"
-      data_tests:
-      - not_null
-    - name: is_open
-      description: "Whether the exchange is open"
-      data_tests:
-      - not_null
-    - name: pretrade_date
-      description: "The previous trading date"
-  - name: stg.tushare_stock_basic
-    columns:
-    - name: ts_code
-      description: TS代码
-      data_tests:
-      - not_null
-    - name: symbol
-      description: 股票代码
-    - name: name
-      description: 股票名称
-    - name: area
-      description: 所在地域
-    - name: industry
-      description: 所属行业
-    - name: fullname
-      description: 股票全称
-    - name: enname
-      description: 英文全称
-    - name: cnspell
-      description: 拼音缩写
-    - name: market
-      description: 市场类型 （主板/中小板/创业板）
-    - name: exchange
-      description: 交易所代码
-      data_tests:
-      - not_null
-    - name: curr_type
-      description: 交易货币
-    - name: list_status
-      description: 上市状态： L上市 D退市 P暂停上市
-    - name: list_date
-      description: 上市日期
-    - name: delist_date
-      description: 退市日期
-    - name: is_hs
-      description: 是否沪深港通标的，N否 H沪股通 S深股通
-    - name: act_name
-      description: 实控人名称
-    - name: act_ent_type
-      description: 实控人企业性质
-    - name: dt
-      description: 最后修改时间
-    unique_composite:
-    - name: unique_exchange_ts_code
       columns:
-      - exchange
+      - name: exchange
-      - ts_code
+        description: "The exchange code"
-  - name: stg.tushare_daily
+        tests:
-    description: "沪深股票交易日线数据"
+          - not_null
-    data_tests:
+      - name: cal_date
-    - dbt_utils.unique_combination_of_columns:
+        description: "The date"
-        combination_of_columns:
+        tests:
-        - ts_code
+          - not_null
-        - trade_date
+      - name: is_open
-    columns:
+        description: "Whether the exchange is open"
-    - name: ts_code
+        tests:
-      description: TS代码
+          - not_null
-      data_tests:
+      - name: pretrade_date
-      - not_null
+        description: "The previous trading date"
-    - name: trade_date
+    - name: stg.tushare_stock_basic
-      description: 交易日期
+      columns:
-      data_tests:
+        - name: ts_code
-      - not_null
+          description: TS代码
-    - name: open
+          tests:
-      description: 开盘价
+            - not_null
-      data_tests:
+        - name: symbol
-      - not_null
+          description: 股票代码
-    - name: high
+        - name: name
-      description: 最高价
+          description: 股票名称
-      data_tests:
+        - name: area
-      - not_null
+          description: 所在地域
-    - name: low
+        - name: industry
-      description: 最低价
+          description: 所属行业
-      data_tests:
+        - name: fullname
-      - not_null
+          description: 股票全称
-    - name: close
+        - name: enname
-      description: 收盘价
+          description: 英文全称
-      data_tests:
+        - name: cnspell
-      - not_null
+          description: 拼音缩写
-    - name: pre_close
+        - name: market
-      description: 昨收价
+          description: 市场类型 （主板/中小板/创业板）
-      data_tests:
+        - name: exchange
-      - not_null
+          description: 交易所代码
-    - name: change
+          tests:
-      description: 涨跌额
+            - not_null
-      data_tests:
+        - name: curr_type
-      - not_null
+          description: 交易货币
-    - name: pct_chg
+        - name: list_status
-      description: 涨跌幅
+          description: 上市状态： L上市 D退市 P暂停上市
-      data_tests:
+        - name: list_date
-      - not_null
+          description: 上市日期
-    - name: vol
+        - name: delist_date
-      description: 成交量
+          description: 退市日期
-      data_tests:
+        - name: is_hs
-      - not_null
+          description: 是否沪深港通标的，N否 H沪股通 S深股通
-    - name: amount
+        - name: act_name
-      description: 成交额
+          description: 实控人名称
-  - name: stg.tushare_daily_basic
+        - name: act_ent_type
-    description: "沪深股票每日指标数据"
+          description: 实控人企业性质
-    data_tests:
+        - name: dt
-    - dbt_utils.unique_combination_of_columns:
+          description: 最后修改时间
-        combination_of_columns:
+      unique_composite:
-        - ts_code
+        - name: unique_exchange_ts_code
-        - trade_date
+          columns:
+            - exchange
+            - ts_code
+    - name: stg.tushare_daily
+      description: "沪深股票交易日线数据"
+      tests:
+        - dbt_utils.unique_combination_of_columns:
+            combination_of_columns:
+              - ts_code
+              - trade_date
+      columns:
+        - name: ts_code
+          description: TS代码
+          tests:
+            - not_null
+        - name: trade_date
+          description: 交易日期
+          tests:
+            - not_null
+        - name: open
+          description: 开盘价
+          tests:
+            - not_null
+        - name: high
+          description: 最高价
+          tests:
+            - not_null
+        - name: low
+          description: 最低价
+          tests:
+            - not_null
+        - name: close
+          description: 收盘价
+          tests:
+            - not_null
+        - name: pre_close
+          description: 昨收价
+          tests:
+            - not_null
+        - name: change
+          description: 涨跌额
+          tests:
+            - not_null
+        - name: pct_chg
+          description: 涨跌幅
+          tests:
+            - not_null
+        - name: vol
+          description: 成交量
+          tests:
+            - not_null
+        - name: amount
+          description: 成交额
+    - name: stg.tushare_daily_basic
+      description: "沪深股票每日指标数据"
+      tests:
+        - dbt_utils.unique_combination_of_columns:
+            combination_of_columns:
+              - ts_code
+              - trade_date
 
-    columns:
+      columns:
-    - name: ts_code
+        - name: ts_code
-      description: "股票代码"
+          description: "股票代码"
-      data_type: String
+          data_type: String
-    - name: trade_date
+        - name: trade_date
-      description: "交易日期"
+          description: "交易日期"
-      data_type: String
+          data_type: String
-    - name: close
+        - name: close
-      description: "当日收盘价"
+          description: "当日收盘价"
-      data_type: Nullable(Float32)
+          data_type: Nullable(Float32)
-    - name: turnover_rate
+        - name: turnover_rate
-      description: "换手率（%）"
+          description: "换手率（%）"
-      data_type: Nullable(Float32)
+          data_type: Nullable(Float32)
-    - name: turnover_rate_f
+        - name: turnover_rate_f
-      description: "换手率（自由流通股）"
+          description: "换手率（自由流通股）"
-      data_type: Nullable(Float32)
+          data_type: Nullable(Float32)
-    - name: volume_ratio
+        - name: volume_ratio
-      description: "量比"
+          description: "量比"
-      data_type: Nullable(Float32)
+          data_type: Nullable(Float32)
-    - name: pe
+        - name: pe
-      description: "市盈率（总市值/净利润， 亏损的PE为空）"
+          description: "市盈率（总市值/净利润， 亏损的PE为空）"
-      data_type: Nullable(Float32)
+          data_type: Nullable(Float32)
-    - name: pe_ttm
+        - name: pe_ttm
-      description: "市盈率（TTM，亏损的PE为空）"
+          description: "市盈率（TTM，亏损的PE为空）"
-      data_type: Nullable(Float32)
+          data_type: Nullable(Float32)
-    - name: pb
+        - name: pb
-      description: "市净率（总市值/净资产）"
+          description: "市净率（总市值/净资产）"
-      data_type: Nullable(Float32)
+          data_type: Nullable(Float32)
-    - name: ps
+        - name: ps
-      description: "市销率"
+          description: "市销率"
-      data_type: Nullable(Float32)
+          data_type: Nullable(Float32)
-    - name: ps_ttm
+        - name: ps_ttm
-      description: "市销率（TTM）"
+          description: "市销率（TTM）"
-      data_type: Nullable(Float32)
+          data_type: Nullable(Float32)
-    - name: dv_ratio
+        - name: dv_ratio
-      description: "股息率 （%）"
+          description: "股息率 （%）"
-      data_type: Nullable(Float32)
+          data_type: Nullable(Float32)
-    - name: dv_ttm
+        - name: dv_ttm
-      description: "股息率（TTM）（%）"
+          description: "股息率（TTM）（%）"
-      data_type: Nullable(Float32)
+          data_type: Nullable(Float32)
-    - name: total_share
+        - name: total_share
-      description: "总股本 （万股）"
+          description: "总股本 （万股）"
-      data_type: Nullable(Float32)
+          data_type: Nullable(Float32)
-    - name: float_share
+        - name: float_share
-      description: "流通股本 （万股）"
+          description: "流通股本 （万股）"
-      data_type: Nullable(Float32)
+          data_type: Nullable(Float32)
-    - name: free_share
+        - name: free_share
-      description: "自由流通股本 （万）"
+          description: "自由流通股本 （万）"
-      data_type: Nullable(Float32)
+          data_type: Nullable(Float32)
-    - name: total_mv
+        - name: total_mv
-      description: "总市值 （万元）"
+          description: "总市值 （万元）"
-      data_type: Nullable(Float32)
+          data_type: Nullable(Float32)
-    - name: circ_mv
+        - name: circ_mv
-      description: "流通市值（万元）"
+          description: "流通市值（万元）"
-      data_type: Nullable(Float32)
+          data_type: Nullable(Float32)
-  - name: stg.tushare_adj_factor
+    - name: stg.tushare_adj_factor
-    description: "沪深股票复权因子"
+      description: "沪深股票复权因子"
-    data_tests:
+      tests:
-    - dbt_utils.unique_combination_of_columns:
+        - dbt_utils.unique_combination_of_columns:
-        combination_of_columns:
+            combination_of_columns:
-        - ts_code
+              - ts_code
-        - trade_date
+              - trade_date
-    columns:
+      columns:
-    - name: ts_code
+        - name: ts_code
-      description: "股票代码"
+          description: "股票代码"
-      data_type: String
+          data_type: String
-    - name: trade_date
+        - name: trade_date
-      description: "交易日期"
+          description: "交易日期"
-      data_type: String
+          data_type: String
-    - name: adj_factor
+        - name: adj_factor
-      description: "复权因子"
+          description: "复权因子"
-      data_type: Nullable(Decimal(16, 6))
+          data_type: Nullable(Decimal(16, 6))
-    - name: dt
+        - name: dt
-      description: "更新时间"
+          description: "更新日期"
-      data_type: DateTime
+          data_type: DateTime
-  - name: stg.tushare_minutes
+    - name: stg.tushare_minutes
-    description: "沪深分钟级交易数据"
+      description: "沪深分钟级交易数据"
-    data_tests:
+      tests:
-    - dbt_utils.unique_combination_of_columns:
+        - dbt_utils.unique_combination_of_columns:
-        combination_of_columns:
+            combination_of_columns:
-        - ts_code
+              - ts_code
-        - trade_time
+              - trade_time
-        config:
+            config:
-          where: "left(trade_time,7)>=concat(cast(year(today())-1 As String), '-01')"
+              where: "left(trade_time,7)>=concat(cast(year(today())-1 As String), '-01')"
-    columns:
+      columns:
-    - name: ts_code
+        - name: ts_code
-      description: "The stock code."
+          description: "The stock code."
-      data_type: String
+          data_type: String
-      data_tests:
+          tests:
-      - not_null
+            - not_null
-    - name: trade_time
+        - name: trade_time
-      description: "The trading time."
+          description: "The trading time."
-      data_type: String
+          data_type: String
-      data_tests:
+          tests:
-      - not_null
+            - not_null
-    - name: close
+        - name: close
-      description: "The closing price."
+          description: "The closing price."
-      data_type: Nullable(Float32)
+          data_type: Nullable(Float32)
-    - name: open
+        - name: open
-      description: "The opening price."
+          description: "The opening price."
-      data_type: Nullable(Float32)
+          data_type: Nullable(Float32)
-    - name: high
+        - name: high
-      description: "The highest price."
+          description: "The highest price."
-      data_type: Nullable(Float32)
+          data_type: Nullable(Float32)
-    - name: low
+        - name: low
-      description: "The lowest price."
+          description: "The lowest price."
-      data_type: Nullable(Float32)
+          data_type: Nullable(Float32)
-    - name: vol
+        - name: vol
-      description: "The volume of trades."
+          description: "The volume of trades."
-      data_type: Nullable(Float32)
+          data_type: Nullable(Float32)
-    - name: amount
+        - name: amount
-      description: "The amount of trades."
+          description: "The amount of trades."
-      data_type: Nullable(Float32)
+          data_type: Nullable(Float32)
-    meta:
+      meta:
-      engine: ReplacingMergeTree
+        engine: ReplacingMergeTree
-      partition_by: "left(trade_time, 7)"
+        partition_by: "left(trade_time, 7)"
-      order_by: "(ts_code, trade_time)"
+        order_by: "(ts_code, trade_time)"
-      settings:
+        settings:
-        index_granularity: 8192
+          index_granularity: 8192

models/ods/ods.tushare_adj_factor.sql

@@ -1,19 +0,0 @@
-{{ 
-    config(
-        materialized='incremental',
-        engine="MergeTree",
-        order_by="(date_id, ts_code)",
-        unique_key=['date_id', 'ts_code'],
-        incremental_strategy='delete+insert',
-    )
-}}
-
-Select
-    ts_code As ts_code,
-    toYYYYMMDD(toDate(trade_date)) As date_id,
-    adj_factor As adj_factor,
-    dt As dt
-From {{ source('finance', 'stg.tushare_adj_factor') }} As StgAdjFactor
-{% if is_incremental() %}
-Where `trade_date` >= (Select formatDateTimeInJodaSyntax(YYYYMMDDToDate(max(`date_id`)), 'yyyy-MM-dd') From {{ this }})
-{% endif %}

models/ods/ods.tushare_minutes.sql

@@ -1,35 +0,0 @@
-{{ 
-    config(
-        materialized='incremental',
-        engine="MergeTree",
-        order_by='(time_id, ts_code)',
-        unique_key=['time_id', 'ts_code'],
-        partition_by=['toYYYYMM(toDateTime(time_id))'],
-        incremental_strategy='delete+insert',
-        query_settings={
-            "join_algorithm": "'full_sorting_merge'", 
-            "max_bytes_before_external_sort": "'1000M'",
-            "max_bytes_before_external_group_by":"'1000M'",
-        }
-    )
-}}
-
-Select 
-    StgMinutes.ts_code As ts_code,
-    toYYYYMMDDhhmmss(toDateTime(`StgMinutes`.`trade_time`)) As time_id,
-    StgMinutes.close As close,
-    StgMinutes.open As open,
-    StgMinutes.high As high,
-    StgMinutes.low As low,
-    StgMinutes.vol As vol,
-    StgMinutes.amount As amount
-From {{ source('finance', 'stg.tushare_minutes') }} As StgMinutes 
-where 1 = 1
-{% if is_incremental() %}
-    And StgMinutes.trade_time >= (Select formatDateTimeInJodaSyntax(toStartOfDay(YYYYMMDDhhmmssToDateTime(max(time_id))), 'yyyy-MM-dd HH:mm:ss') From {{ this }})
-{% endif %}
-{% if 'dev' in target.name %}
-    And StgMinutes.trade_time >= formatDateTimeInJodaSyntax(dateAdd(Month,-1,today()), 'yyyy-01-01 00:00:00')
-{% elif target.name == 'test' %}
-    And StgMinutes.trade_time >= formatDateTimeInJodaSyntax(dateAdd(Year,-1,today()), 'yyyy-01-01 00:00:00')
-{% endif %}

models/ods/schema.yml

@@ -1,54 +0,0 @@
-
-version: 2
-
-models:
-- name: ods.tushare_minutes
-  description: "沪深股票交易分钟线数据"
-  data_tests:
-  - dbt_utils.unique_combination_of_columns:
-      combination_of_columns:
-      - ts_code
-      - time_id
-  columns:
-  - name: ts_code
-    description: TS代码
-    data_tests:
-    - not_null
-  - name: time_id
-    description: 交易日期
-    data_tests:
-    - not_null
-  - name: close
-    description: 收盘价
-  - name: open
-    description: 开盘价
-  - name: high
-    description: 最高价
-  - name: low
-    description: 最低价
-  - name: vol
-    description: 成交量
-  - name: amount
-    description: 成交额
-  - name: dt
-    description: 更新时间
-- name: ods.tushare_adj_factor
-  description: "沪深股票复权因子"
-  data_tests:
-  - dbt_utils.unique_combination_of_columns:
-      combination_of_columns:
-      - ts_code
-      - date_id
-  columns:
-  - name: ts_code
-    description: TS代码
-    data_tests:
-    - not_null
-  - name: date_id
-    description: 交易日期
-    data_tests:
-    - not_null
-  - name: adj_factor
-    description: 复权因子
-  - name: dt
-    description: 更新时间

profiles.yml

@@ -15,7 +15,7 @@ finance_dbt:
       retries: 1
       compression: gzip
       connect_timeout: 10
-      send_receive_timeout: 24000
+      send_receive_timeout: 300
       cluster_mode: False
       use_lw_deletes: True
       check_exchange: True

requirements.txt

@@ -1 +1,2 @@
-dbt-clickhouse==1.8.0
+dbt==1.0.0.37.0
+dbt-clickhouse==1.7.3