diff --git a/models/dw/dw.dim_date.sql b/models/dw/dw.dim_date.sql index 75a4f53..7d16152 100644 --- a/models/dw/dw.dim_date.sql +++ b/models/dw/dw.dim_date.sql @@ -10,7 +10,8 @@ ) }} SELECT - toUInt32(year(date) * 10000 + month(date) * 100 + day(date)) AS date_id, + toYYYYMMDD(date) AS date_id, + formatDateTimeInJodaSyntax(date, 'yyyyMMdd') AS date_id_str, date AS full_date, toYear(date) AS year, toUInt8(quarter(date)) AS quarter, diff --git a/models/dw/dw.fact_stock_daily.sql b/models/dw/dw.fact_stock_daily.sql index ff6f4c7..9812134 100644 --- a/models/dw/dw.fact_stock_daily.sql +++ b/models/dw/dw.fact_stock_daily.sql @@ -1,10 +1,10 @@ {{ config( materialized='incremental', - engine='ReplacingMergeTree', + engine='MergeTree', order_by='(date_id, ts_code)', unique_key=['date_id', 'ts_code'], - incremental_strategy='append', + incremental_strategy='delete+insert', query_settings={ "join_algorithm": "'full_sorting_merge'", } diff --git a/models/dw/dw.fact_stock_minute.sql b/models/dw/dw.fact_stock_minute.sql index c98f805..a14578c 100644 --- a/models/dw/dw.fact_stock_minute.sql +++ b/models/dw/dw.fact_stock_minute.sql @@ -1,7 +1,7 @@ {{ config( materialized='incremental', - engine='ReplacingMergeTree', + engine='MergeTree', order_by='(time_id, ts_code)', unique_key=['time_id', 'ts_code'], incremental_strategy='append', diff --git a/models/dw/schema.yml b/models/dw/schema.yml index 535fb0e..af28803 100644 --- a/models/dw/schema.yml +++ b/models/dw/schema.yml @@ -7,22 +7,22 @@ models: columns: - name: exchange_id description: "The primary key for this table" - tests: + data_tests: - unique - not_null - name: ts_exchange_code description: "交易所代码" - tests: + data_tests: - not_null - unique - name: exchange_name description: "交易所名称" - tests: + data_tests: - not_null - unique - name: dw.dim_date description: "日期" - tests: + data_tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: - year @@ -31,41 +31,46 @@ models: columns: - name: date_id description: "日期键" - tests: + data_tests: - unique - not_null + - name: date_id_str + description: "日期字符串（yyyyMMdd）" + data_tests: + - not_null + - unique - name: full_date description: "日期" - tests: + data_tests: - not_null - unique - name: year description: "年" - tests: + data_tests: - not_null - name: quarter description: "季度" - tests: + data_tests: - not_null - name: month description: "月" - tests: + data_tests: - not_null - name: day description: "日" - tests: + data_tests: - not_null - name: week_num_of_year description: "年中第几周" - tests: + data_tests: - not_null - name: day_num_of_week description: "星期" - tests: + data_tests: - not_null - name: dw.dim_time description: "时间" - tests: + data_tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: - date_id @@ -74,38 +79,38 @@ models: columns: - name: time_id description: "时间键" - tests: + data_tests: - unique - not_null - name: full_time description: "时间" - tests: + data_tests: - not_null - unique - name: date_id description: "日期键" - tests: + data_tests: - not_null - name: hour description: "小时" - tests: + data_tests: - not_null - name: minute description: "分钟" - tests: + data_tests: - not_null - name: full_time_str description: "时间字符串（yyyy-MM-dd HH:mm:ss）" - tests: + data_tests: - not_null - unique - name: date_id_str - description: "日期字符串（yyyy-MM-dd）" - tests: + description: "日期字符串（yyyyMMdd）" + data_tests: - not_null - name: dw.dim_hs_calendar description: "沪深交易日历" - tests: + data_tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: - exchange_id @@ -113,19 +118,19 @@ models: columns: - name: exchange_id description: "The primary key for this table" - tests: + data_tests: - not_null - name: date_id description: "交易日期键" - tests: + data_tests: - not_null - name: is_open description: "是否交易" - tests: + data_tests: - not_null - name: dw.dim_stock description: "股票" - tests: + data_tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: - exchange_id @@ -133,34 +138,34 @@ models: columns: - name: ts_code description: "tushare 股票编码" - tests: + data_tests: - unique - not_null - name: exchange_id description: "交易所键" - tests: + data_tests: - not_null - name: code description: "股票代码" - tests: + data_tests: - not_null - name: name description: "股票名称" - tests: + data_tests: - not_null - unique - name: full_name description: "股票全称" - tests: + data_tests: - not_null - unique - name: name_en description: "股票英文名称" - tests: + data_tests: - not_null - name: cn_spell description: "拼音缩写" - tests: + data_tests: - not_null - name: area description: "地区" @@ -176,11 +181,11 @@ models: description: "退市日期" - name: is_active description: "当前是否在市" - tests: + data_tests: - not_null - name: is_hongkong_connect description: "是否沪深通标的" - tests: + data_tests: - not_null - name: actual_controller description: "实际控制人" @@ -190,7 +195,7 @@ models: description: "更新时间" - name: dw.fact_stock_daily description: "股票日数据" - tests: + data_tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: - ts_code @@ -198,11 +203,11 @@ models: columns: - name: ts_code description: "股票键" - tests: + data_tests: - not_null - name: date_id description: "日期键" - tests: + data_tests: - not_null - name: open description: "开盘价" @@ -256,7 +261,7 @@ models: description: "流通市值（万元）" - name: dw.fact_stock_minute description: "股票分钟数据" - tests: + data_tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: - ts_code @@ -265,12 +270,12 @@ models: - name: ts_code description: "股票键" data_type: "String" - tests: + data_tests: - not_null - name: time_id description: "时间键" data_type: "Int64" - tests: + data_tests: - not_null - name: open description: "开盘价" diff --git a/models/finance_source.yml b/models/finance_source.yml index 400eda1..3d90ecc 100644 --- a/models/finance_source.yml +++ b/models/finance_source.yml @@ -7,7 +7,7 @@ sources: - name: stg.tushare_trade_calendar description: > The source table for the trade calendar data from Tushare - tests: + data_tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: - exchange @@ -15,15 +15,15 @@ sources: columns: - name: exchange description: "The exchange code" - tests: + data_tests: - not_null - name: cal_date description: "The date" - tests: + data_tests: - not_null - name: is_open description: "Whether the exchange is open" - tests: + data_tests: - not_null - name: pretrade_date description: "The previous trading date" @@ -31,7 +31,7 @@ sources: columns: - name: ts_code description: TS代码 - tests: + data_tests: - not_null - name: symbol description: 股票代码 @@ -51,7 +51,7 @@ sources: description: 市场类型 （主板/中小板/创业板） - name: exchange description: 交易所代码 - tests: + data_tests: - not_null - name: curr_type description: 交易货币 @@ -76,7 +76,7 @@ sources: - ts_code - name: stg.tushare_daily description: "沪深股票交易日线数据" - tests: + data_tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: - ts_code @@ -84,49 +84,49 @@ sources: columns: - name: ts_code description: TS代码 - tests: + data_tests: - not_null - name: trade_date description: 交易日期 - tests: + data_tests: - not_null - name: open description: 开盘价 - tests: + data_tests: - not_null - name: high description: 最高价 - tests: + data_tests: - not_null - name: low description: 最低价 - tests: + data_tests: - not_null - name: close description: 收盘价 - tests: + data_tests: - not_null - name: pre_close description: 昨收价 - tests: + data_tests: - not_null - name: change description: 涨跌额 - tests: + data_tests: - not_null - name: pct_chg description: 涨跌幅 - tests: + data_tests: - not_null - name: vol description: 成交量 - tests: + data_tests: - not_null - name: amount description: 成交额 - name: stg.tushare_daily_basic description: "沪深股票每日指标数据" - tests: + data_tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: - ts_code @@ -189,7 +189,7 @@ sources: data_type: Nullable(Float32) - name: stg.tushare_adj_factor description: "沪深股票复权因子" - tests: + data_tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: - ts_code @@ -209,7 +209,7 @@ sources: data_type: DateTime - name: stg.tushare_minutes description: "沪深分钟级交易数据" - tests: + data_tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: - ts_code @@ -220,12 +220,12 @@ sources: - name: ts_code description: "The stock code." data_type: String - tests: + data_tests: - not_null - name: trade_time description: "The trading time." data_type: String - tests: + data_tests: - not_null - name: close description: "The closing price." diff --git a/models/ods/ods.tushare_adj_factor.sql b/models/ods/ods.tushare_adj_factor.sql index c8df741..83bdd40 100644 --- a/models/ods/ods.tushare_adj_factor.sql +++ b/models/ods/ods.tushare_adj_factor.sql @@ -1,10 +1,10 @@ {{ config( materialized='incremental', - engine="ReplacingMergeTree", + engine="MergeTree", order_by="(date_id, ts_code)", unique_key=['date_id', 'ts_code'], - incremental_strategy='append', + incremental_strategy='delete+insert', ) }} diff --git a/models/ods/ods.tushare_minutes.sql b/models/ods/ods.tushare_minutes.sql index bb3c547..13a0cdc 100644 --- a/models/ods/ods.tushare_minutes.sql +++ b/models/ods/ods.tushare_minutes.sql @@ -1,11 +1,11 @@ {{ config( materialized='incremental', - engine="ReplacingMergeTree", + engine="MergeTree", order_by='(time_id, ts_code)', unique_key=['time_id', 'ts_code'], partition_by=['toYYYYMM(toDateTime(time_id))'], - incremental_strategy='append', + incremental_strategy='delete+insert', query_settings={ "join_algorithm": "'full_sorting_merge'", "max_bytes_before_external_sort": "'1000M'", diff --git a/models/ods/schema.yml b/models/ods/schema.yml index e300d86..032e035 100644 --- a/models/ods/schema.yml +++ b/models/ods/schema.yml @@ -4,7 +4,7 @@ version: 2 models: - name: ods.tushare_minutes description: "沪深股票交易分钟线数据" - tests: + data_tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: - ts_code @@ -12,11 +12,11 @@ models: columns: - name: ts_code description: TS代码 - tests: + data_tests: - not_null - name: time_id description: 交易日期 - tests: + data_tests: - not_null - name: close description: 收盘价 @@ -34,7 +34,7 @@ models: description: 更新时间 - name: ods.tushare_adj_factor description: "沪深股票复权因子" - tests: + data_tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: - ts_code @@ -42,11 +42,11 @@ models: columns: - name: ts_code description: TS代码 - tests: + data_tests: - not_null - name: date_id description: 交易日期 - tests: + data_tests: - not_null - name: adj_factor description: 复权因子