Compare commits
No commits in common. "dev" and "debug" have entirely different histories.
@ -4,7 +4,7 @@ WORKDIR /app
|
||||
### Pip
|
||||
USER root
|
||||
COPY ./requirements.txt /app/
|
||||
RUN pip config set global.index-url https://nexus.mujiannan.com:5001/repository/pypiserver/simple && \
|
||||
RUN pip3 config set global.index-url https://nexus.mujiannan.com:5001/repository/pypiserver/simple && \
|
||||
pip install -r requirements.txt
|
||||
USER 1001
|
||||
## Copy files
|
||||
|
||||
@ -1,2 +1,2 @@
|
||||
dbt docs generate
|
||||
dbt docs serve --host=0.0.0.0 --port=8080
|
||||
dbt docs serve
|
||||
@ -10,8 +10,7 @@
|
||||
)
|
||||
}}
|
||||
SELECT
|
||||
toYYYYMMDD(date) AS date_id,
|
||||
formatDateTimeInJodaSyntax(date, 'yyyyMMdd') AS date_id_str,
|
||||
toUInt32(year(date) * 10000 + month(date) * 100 + day(date)) AS date_id,
|
||||
date AS full_date,
|
||||
toYear(date) AS year,
|
||||
toUInt8(quarter(date)) AS quarter,
|
||||
|
||||
@ -2,16 +2,14 @@
|
||||
config(
|
||||
materialized='table',
|
||||
engine='MergeTree',
|
||||
order_by='time_id',
|
||||
order_by='time_id'
|
||||
)
|
||||
}}
|
||||
Select toYYYYMMDDhhmmss(`full_time`) As time_id
|
||||
Select cast(formatDateTime(`full_time`, '%Y%m%d%H%i') As Int64) As time_id
|
||||
, `full_time`
|
||||
, `date_id`
|
||||
, hour(`full_time`) As `hour`
|
||||
, minute(`full_time`) As `minute`
|
||||
, formatDateTimeInJodaSyntax(`full_time`, 'yyyy-MM-dd HH:mm:ss') As full_time_str
|
||||
, Cast(`date_id` As String) As date_id_str
|
||||
From(
|
||||
SELECT
|
||||
arrayJoin(
|
||||
|
||||
@ -2,12 +2,9 @@
|
||||
config(
|
||||
materialized='incremental',
|
||||
engine='MergeTree',
|
||||
order_by='(date_id, ts_code)',
|
||||
unique_key=['date_id', 'ts_code'],
|
||||
order_by='ts_code, date_id',
|
||||
unique_key=['ts_code', 'date_id'],
|
||||
incremental_strategy='delete+insert',
|
||||
query_settings={
|
||||
"join_algorithm": "'full_sorting_merge'",
|
||||
}
|
||||
)
|
||||
}}
|
||||
Select
|
||||
@ -38,7 +35,7 @@ Select
|
||||
StgDailyBasic.free_share As free_share,
|
||||
StgDailyBasic.total_mv As total_mv,
|
||||
StgDailyBasic.circ_mv As circ_mv
|
||||
From {{ source('finance', 'stg.tushare_daily') }} As StgDaily
|
||||
From {{ source('finance', 'stg.tushare_daily') }} As StgDaily Final
|
||||
Inner Join {{ ref('dw.dim_date') }} As DimDate
|
||||
On StgDaily.trade_date = Cast(DimDate.date_id As String)
|
||||
Any Left Join {{ source('finance', 'stg.tushare_daily_basic') }} As StgDailyBasic
|
||||
|
||||
@ -1,40 +1,34 @@
|
||||
{{
|
||||
config(
|
||||
materialized='incremental',
|
||||
materialized='materialized_view',
|
||||
engine='MergeTree',
|
||||
order_by='(time_id, ts_code)',
|
||||
unique_key=['time_id', 'ts_code'],
|
||||
incremental_strategy='delete+insert',
|
||||
partition_by=['toYYYYMM(toDateTime(time_id))'],
|
||||
query_settings={
|
||||
"join_algorithm": "'full_sorting_merge'",
|
||||
}
|
||||
order_by='ts_code, time_id',
|
||||
unique_key=['ts_code', 'time_id'],
|
||||
partition_by=['cast(floor(time_id / 1000000) As Int32)'],
|
||||
query_settings={"join_algorithm": "'partial_merge'"}
|
||||
)
|
||||
}}
|
||||
|
||||
Select
|
||||
OdsMinutes.ts_code As ts_code,
|
||||
OdsMinutes.time_id As time_id,
|
||||
OdsMinutes.close As close,
|
||||
OdsMinutes.open As open,
|
||||
OdsMinutes.high As high,
|
||||
OdsMinutes.low As low,
|
||||
OdsMinutes.vol As vol,
|
||||
OdsMinutes.amount As amount,
|
||||
OdsAdjFactor.adj_factor As adj_factor,
|
||||
StgMinutes.ts_code As ts_code,
|
||||
DimTime.time_id As time_id,
|
||||
StgMinutes.close As close,
|
||||
StgMinutes.open As open,
|
||||
StgMinutes.high As high,
|
||||
StgMinutes.low As low,
|
||||
StgMinutes.vol As vol,
|
||||
StgMinutes.amount As amount,
|
||||
StgAdjFactor.adj_factor As adj_factor,
|
||||
now() As dt
|
||||
From {{ ref('ods.tushare_minutes') }} As OdsMinutes
|
||||
Any Left Join {{ ref('dw.dim_time') }} As DimTime
|
||||
On OdsMinutes.time_id = DimTime.time_id
|
||||
Any Left Join {{ ref('ods.tushare_adj_factor') }} As OdsAdjFactor
|
||||
On OdsMinutes.ts_code = OdsAdjFactor.ts_code
|
||||
And OdsAdjFactor.date_id = DimTime.date_id
|
||||
From {{ source('finance', 'stg.tushare_minutes') }} As StgMinutes
|
||||
Inner Join {{ ref('dw.dim_time') }} As DimTime
|
||||
On StgMinutes.trade_time = formatDateTimeInJodaSyntax(full_time, 'yyyy-MM-dd HH:mm:ss')
|
||||
Any Left Join {{ source('finance', 'stg.tushare_adj_factor') }} As StgAdjFactor
|
||||
On StgMinutes.ts_code = StgAdjFactor.ts_code
|
||||
And StgAdjFactor.trade_date = Cast(DimTime.date_id As String)
|
||||
where 1 = 1
|
||||
{% if is_incremental() %}
|
||||
And OdsMinutes.time_id >= (Select toYYYYMMDDhhmmss(toStartOfDay(YYYYMMDDhhmmssToDateTime(max(`time_id`)))) From {{ this }})
|
||||
{% endif %}
|
||||
{% if 'dev' in target.name %}
|
||||
And OdsMinutes.time_id >= toYYYYMMDDhhmmss(dateAdd(Month,-1,today()))
|
||||
And StgMinutes.trade_time >= formatDateTimeInJodaSyntax(today(), 'yyyy-01-01 00:00:00')
|
||||
{% elif target.name == 'test' %}
|
||||
And OdsMinutes.time_id >= toYYYYMMDDhhmmss(dateAdd(Year,-1,today()))
|
||||
And StgMinutes.trade_time >= formatDateTimeInJodaSyntax(dateAdd(Year,-1,today()), 'yyyy-01-01 00:00:00')
|
||||
{% endif %}
|
||||
@ -2,303 +2,291 @@
|
||||
version: 2
|
||||
|
||||
models:
|
||||
- name: dw.dim_exchange
|
||||
description: "交易所"
|
||||
columns:
|
||||
- name: exchange_id
|
||||
description: "The primary key for this table"
|
||||
data_tests:
|
||||
- unique
|
||||
- not_null
|
||||
- name: ts_exchange_code
|
||||
description: "交易所代码"
|
||||
data_tests:
|
||||
- not_null
|
||||
- unique
|
||||
- name: exchange_name
|
||||
description: "交易所名称"
|
||||
data_tests:
|
||||
- not_null
|
||||
- unique
|
||||
- name: dw.dim_date
|
||||
description: "日期"
|
||||
data_tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- year
|
||||
- month
|
||||
- day
|
||||
columns:
|
||||
- name: date_id
|
||||
description: "日期键"
|
||||
data_tests:
|
||||
- unique
|
||||
- not_null
|
||||
- name: date_id_str
|
||||
description: "日期字符串（yyyyMMdd）"
|
||||
data_tests:
|
||||
- not_null
|
||||
- unique
|
||||
- name: full_date
|
||||
- name: dw.dim_exchange
|
||||
description: "交易所"
|
||||
columns:
|
||||
- name: exchange_id
|
||||
description: "The primary key for this table"
|
||||
tests:
|
||||
- unique
|
||||
- not_null
|
||||
- name: ts_exchange_code
|
||||
description: "交易所代码"
|
||||
tests:
|
||||
- not_null
|
||||
- unique
|
||||
- name: exchange_name
|
||||
description: "交易所名称"
|
||||
tests:
|
||||
- not_null
|
||||
- unique
|
||||
- name: dw.dim_date
|
||||
description: "日期"
|
||||
data_tests:
|
||||
- not_null
|
||||
- unique
|
||||
- name: year
|
||||
description: "年"
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: quarter
|
||||
description: "季度"
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: month
|
||||
description: "月"
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: day
|
||||
description: "日"
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: week_num_of_year
|
||||
description: "年中第几周"
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: day_num_of_week
|
||||
description: "星期"
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: dw.dim_time
|
||||
description: "时间"
|
||||
data_tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- date_id
|
||||
- hour
|
||||
- minute
|
||||
columns:
|
||||
- name: time_id
|
||||
description: "时间键"
|
||||
data_tests:
|
||||
- unique
|
||||
- not_null
|
||||
- name: full_time
|
||||
tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- year
|
||||
- month
|
||||
- day
|
||||
columns:
|
||||
- name: date_id
|
||||
description: "日期键"
|
||||
tests:
|
||||
- unique
|
||||
- not_null
|
||||
- name: full_date
|
||||
description: "日期"
|
||||
tests:
|
||||
- not_null
|
||||
- unique
|
||||
- name: year
|
||||
description: "年"
|
||||
tests:
|
||||
- not_null
|
||||
- name: quarter
|
||||
description: "季度"
|
||||
tests:
|
||||
- not_null
|
||||
- name: month
|
||||
description: "月"
|
||||
tests:
|
||||
- not_null
|
||||
- name: day
|
||||
description: "日"
|
||||
tests:
|
||||
- not_null
|
||||
- name: week_num_of_year
|
||||
description: "年中第几周"
|
||||
tests:
|
||||
- not_null
|
||||
- name: day_num_of_week
|
||||
description: "星期"
|
||||
tests:
|
||||
- not_null
|
||||
- name: dw.dim_time
|
||||
description: "时间"
|
||||
data_tests:
|
||||
- not_null
|
||||
- unique
|
||||
- name: date_id
|
||||
description: "日期键"
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: hour
|
||||
description: "小时"
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: minute
|
||||
description: "分钟"
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: full_time_str
|
||||
description: "时间字符串（yyyy-MM-dd HH:mm:ss）"
|
||||
data_tests:
|
||||
- not_null
|
||||
- unique
|
||||
- name: date_id_str
|
||||
description: "日期字符串（yyyyMMdd）"
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: dw.dim_hs_calendar
|
||||
description: "沪深交易日历"
|
||||
data_tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- exchange_id
|
||||
- date_id
|
||||
columns:
|
||||
- name: exchange_id
|
||||
description: "The primary key for this table"
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: date_id
|
||||
description: "交易日期键"
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: is_open
|
||||
description: "是否交易"
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: dw.dim_stock
|
||||
description: "股票"
|
||||
data_tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- exchange_id
|
||||
- code
|
||||
columns:
|
||||
- name: ts_code
|
||||
description: "tushare 股票编码"
|
||||
data_tests:
|
||||
- unique
|
||||
- not_null
|
||||
- name: exchange_id
|
||||
description: "交易所键"
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: code
|
||||
description: "股票代码"
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: name
|
||||
description: "股票名称"
|
||||
data_tests:
|
||||
- not_null
|
||||
- unique
|
||||
- name: full_name
|
||||
description: "股票全称"
|
||||
data_tests:
|
||||
- not_null
|
||||
- unique
|
||||
- name: name_en
|
||||
description: "股票英文名称"
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: cn_spell
|
||||
description: "拼音缩写"
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: area
|
||||
description: "地区"
|
||||
- name: industry
|
||||
description: "行业"
|
||||
- name: market
|
||||
description: "市场类型 （主板/中小板/创业板）"
|
||||
- name: currency
|
||||
description: "交易货币"
|
||||
- name: list_date
|
||||
description: "上市日期"
|
||||
- name: delist_date
|
||||
description: "退市日期"
|
||||
- name: is_active
|
||||
description: "当前是否在市"
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: is_hongkong_connect
|
||||
description: "是否沪深通标的"
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: actual_controller
|
||||
description: "实际控制人"
|
||||
- name: actual_controller_enterprise_type
|
||||
description: "实际控制人企业类型"
|
||||
- name: dt
|
||||
description: "更新时间"
|
||||
- name: dw.fact_stock_daily
|
||||
description: "股票日数据"
|
||||
data_tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- ts_code
|
||||
- date_id
|
||||
columns:
|
||||
- name: ts_code
|
||||
description: "股票键"
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: date_id
|
||||
description: "日期键"
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: open
|
||||
description: "开盘价"
|
||||
- name: high
|
||||
description: "最高价"
|
||||
- name: low
|
||||
description: "最低价"
|
||||
- name: close
|
||||
description: "收盘价"
|
||||
- name: pre_close
|
||||
description: "昨收价"
|
||||
- name: change
|
||||
description: "涨跌额"
|
||||
- name: pct_chg
|
||||
description: "涨跌幅"
|
||||
- name: vol
|
||||
description: "成交量"
|
||||
- name: amount
|
||||
description: "成交额"
|
||||
- name: adj_factor
|
||||
description: "复权因子"
|
||||
- name: turnover_rate
|
||||
description: "换手率"
|
||||
- name: turnover_rate_free
|
||||
description: "换手率（自由流通股）"
|
||||
- name: volume_ratio
|
||||
description: "量比"
|
||||
- name: pe
|
||||
description: "市盈率（总市值/净利润， 亏损的PE为空）"
|
||||
- name: pe_ttm
|
||||
description: "市盈率（TTM）"
|
||||
- name: pb
|
||||
description: "市净率（总市值/净资产）"
|
||||
- name: ps
|
||||
description: "市销率"
|
||||
- name: ps_ttm
|
||||
description: "市销率（TTM）"
|
||||
- name: dv_ratio
|
||||
description: "股息率"
|
||||
- name: dv_ttm
|
||||
description: "股息率（TTM）"
|
||||
- name: total_share
|
||||
description: "总股本（万股）"
|
||||
- name: float_share
|
||||
description: "流通股本（万股）"
|
||||
- name: free_share
|
||||
description: "自由流通股本（万）"
|
||||
- name: total_mv
|
||||
description: "总市值（万元）"
|
||||
- name: circ_mv
|
||||
description: "流通市值（万元）"
|
||||
- name: dw.fact_stock_minute
|
||||
description: "股票分钟数据"
|
||||
data_tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- ts_code
|
||||
- time_id
|
||||
columns:
|
||||
- name: ts_code
|
||||
description: "股票键"
|
||||
data_type: "String"
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: time_id
|
||||
description: "时间键"
|
||||
data_type: "Int64"
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: open
|
||||
description: "开盘价"
|
||||
data_type: "Nullable(Decimal(16,2))"
|
||||
- name: high
|
||||
description: "最高价"
|
||||
data_type: "Nullable(Decimal(16,2))"
|
||||
- name: low
|
||||
description: "最低价"
|
||||
data_type: "Nullable(Decimal(16,2))"
|
||||
- name: close
|
||||
description: "收盘价"
|
||||
data_type: "Nullable(Decimal(16,2))"
|
||||
- name: vol
|
||||
description: "成交量"
|
||||
data_type: "Nullable(Int32)"
|
||||
- name: amount
|
||||
description: "成交额"
|
||||
data_type: "Nullable(Decimal(18,2))"
|
||||
- name: adj_factor
|
||||
description: "复权因子"
|
||||
data_type: "Decimal(16,6)"
|
||||
- name: dt
|
||||
description: "时间"
|
||||
data_type: "DateTime"
|
||||
|
||||
tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- date_id
|
||||
- hour
|
||||
- minute
|
||||
columns:
|
||||
- name: time_id
|
||||
description: "时间键"
|
||||
tests:
|
||||
- unique
|
||||
- not_null
|
||||
- name: full_time
|
||||
description: "时间"
|
||||
tests:
|
||||
- not_null
|
||||
- unique
|
||||
- name: date_id
|
||||
description: "日期键"
|
||||
tests:
|
||||
- not_null
|
||||
- name: hour
|
||||
description: "小时"
|
||||
tests:
|
||||
- not_null
|
||||
- name: minute
|
||||
description: "分钟"
|
||||
tests:
|
||||
- not_null
|
||||
- name: dw.dim_hs_calendar
|
||||
description: "沪深交易日历"
|
||||
tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- exchange_id
|
||||
- date_id
|
||||
columns:
|
||||
- name: exchange_id
|
||||
description: "The primary key for this table"
|
||||
tests:
|
||||
- not_null
|
||||
- name: date_id
|
||||
description: "交易日期键"
|
||||
tests:
|
||||
- not_null
|
||||
- name: is_open
|
||||
description: "是否交易"
|
||||
tests:
|
||||
- not_null
|
||||
- name: dw.dim_stock
|
||||
description: "股票"
|
||||
tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- exchange_id
|
||||
- code
|
||||
columns:
|
||||
- name: ts_code
|
||||
description: "tushare 股票编码"
|
||||
tests:
|
||||
- unique
|
||||
- not_null
|
||||
- name: exchange_id
|
||||
description: "交易所键"
|
||||
tests:
|
||||
- not_null
|
||||
- name: code
|
||||
description: "股票代码"
|
||||
tests:
|
||||
- not_null
|
||||
- name: name
|
||||
description: "股票名称"
|
||||
tests:
|
||||
- not_null
|
||||
- unique
|
||||
- name: full_name
|
||||
description: "股票全称"
|
||||
tests:
|
||||
- not_null
|
||||
- unique
|
||||
- name: name_en
|
||||
description: "股票英文名称"
|
||||
tests:
|
||||
- not_null
|
||||
- name: cn_spell
|
||||
description: "拼音缩写"
|
||||
tests:
|
||||
- not_null
|
||||
- name: area
|
||||
description: "地区"
|
||||
- name: industry
|
||||
description: "行业"
|
||||
- name: market
|
||||
description: "市场类型 （主板/中小板/创业板）"
|
||||
- name: currency
|
||||
description: "交易货币"
|
||||
- name: list_date
|
||||
description: "上市日期"
|
||||
- name: delist_date
|
||||
description: "退市日期"
|
||||
- name: is_active
|
||||
description: "当前是否在市"
|
||||
tests:
|
||||
- not_null
|
||||
- name: is_hongkong_connect
|
||||
description: "是否沪深通标的"
|
||||
tests:
|
||||
- not_null
|
||||
- name: actual_controller
|
||||
description: "实际控制人"
|
||||
- name: actual_controller_enterprise_type
|
||||
description: "实际控制人企业类型"
|
||||
- name: dt
|
||||
description: "更新时间"
|
||||
- name: dw.fact_stock_daily
|
||||
description: "股票日数据"
|
||||
tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- ts_code
|
||||
- date_id
|
||||
columns:
|
||||
- name: ts_code
|
||||
description: "股票键"
|
||||
tests:
|
||||
- not_null
|
||||
- name: date_id
|
||||
description: "日期键"
|
||||
tests:
|
||||
- not_null
|
||||
- name: open
|
||||
description: "开盘价"
|
||||
- name: high
|
||||
description: "最高价"
|
||||
- name: low
|
||||
description: "最低价"
|
||||
- name: close
|
||||
description: "收盘价"
|
||||
- name: pre_close
|
||||
description: "昨收价"
|
||||
- name: change
|
||||
description: "涨跌额"
|
||||
- name: pct_chg
|
||||
description: "涨跌幅"
|
||||
- name: vol
|
||||
description: "成交量"
|
||||
- name: amount
|
||||
description: "成交额"
|
||||
- name: adj_factor
|
||||
description: "复权因子"
|
||||
- name: turnover_rate
|
||||
description: "换手率"
|
||||
- name: turnover_rate_free
|
||||
description: "换手率（自由流通股）"
|
||||
- name: volume_ratio
|
||||
description: "量比"
|
||||
- name: pe
|
||||
description: "市盈率（总市值/净利润， 亏损的PE为空）"
|
||||
- name: pe_ttm
|
||||
description: "市盈率（TTM）"
|
||||
- name: pb
|
||||
description: "市净率（总市值/净资产）"
|
||||
- name: ps
|
||||
description: "市销率"
|
||||
- name: ps_ttm
|
||||
description: "市销率（TTM）"
|
||||
- name: dv_ratio
|
||||
description: "股息率"
|
||||
- name: dv_ttm
|
||||
description: "股息率（TTM）"
|
||||
- name: total_share
|
||||
description: "总股本（万股）"
|
||||
- name: float_share
|
||||
description: "流通股本（万股）"
|
||||
- name: free_share
|
||||
description: "自由流通股本（万）"
|
||||
- name: total_mv
|
||||
description: "总市值（万元）"
|
||||
- name: circ_mv
|
||||
description: "流通市值（万元）"
|
||||
- name: dw.fact_stock_minute
|
||||
description: "股票分钟数据"
|
||||
tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- ts_code
|
||||
- time_id
|
||||
columns:
|
||||
- name: ts_code
|
||||
description: "股票键"
|
||||
data_type: "String"
|
||||
tests:
|
||||
- not_null
|
||||
- name: time_id
|
||||
description: "时间键"
|
||||
data_type: "Int64"
|
||||
tests:
|
||||
- not_null
|
||||
- name: open
|
||||
description: "开盘价"
|
||||
data_type: "Nullable(Decimal(16,2))"
|
||||
- name: high
|
||||
description: "最高价"
|
||||
data_type: "Nullable(Decimal(16,2))"
|
||||
- name: low
|
||||
description: "最低价"
|
||||
data_type: "Nullable(Decimal(16,2))"
|
||||
- name: close
|
||||
description: "收盘价"
|
||||
data_type: "Nullable(Decimal(16,2))"
|
||||
- name: vol
|
||||
description: "成交量"
|
||||
data_type: "Nullable(Int32)"
|
||||
- name: amount
|
||||
description: "成交额"
|
||||
data_type: "Nullable(Decimal(18,2))"
|
||||
- name: adj_factor
|
||||
description: "复权因子"
|
||||
data_type: "Decimal(16,6)"
|
||||
tests:
|
||||
- not_null
|
||||
- name: dt
|
||||
description: "时间"
|
||||
data_type: "DateTime"
|
||||
|
||||
@ -4,250 +4,250 @@ sources:
|
||||
schema: '{{ env_var("FINANCE_CLICKHOUSE_DATABASE") }}'
|
||||
description: "The source for financial data"
|
||||
tables:
|
||||
- name: stg.tushare_trade_calendar
|
||||
description: >
|
||||
The source table for the trade calendar data from Tushare
|
||||
data_tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- exchange
|
||||
- cal_date
|
||||
columns:
|
||||
- name: exchange
|
||||
description: "The exchange code"
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: cal_date
|
||||
description: "The date"
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: is_open
|
||||
description: "Whether the exchange is open"
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: pretrade_date
|
||||
description: "The previous trading date"
|
||||
- name: stg.tushare_stock_basic
|
||||
columns:
|
||||
- name: ts_code
|
||||
description: TS代码
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: symbol
|
||||
description: 股票代码
|
||||
- name: name
|
||||
description: 股票名称
|
||||
- name: area
|
||||
description: 所在地域
|
||||
- name: industry
|
||||
description: 所属行业
|
||||
- name: fullname
|
||||
description: 股票全称
|
||||
- name: enname
|
||||
description: 英文全称
|
||||
- name: cnspell
|
||||
description: 拼音缩写
|
||||
- name: market
|
||||
description: 市场类型 （主板/中小板/创业板）
|
||||
- name: exchange
|
||||
description: 交易所代码
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: curr_type
|
||||
description: 交易货币
|
||||
- name: list_status
|
||||
description: 上市状态： L上市 D退市 P暂停上市
|
||||
- name: list_date
|
||||
description: 上市日期
|
||||
- name: delist_date
|
||||
description: 退市日期
|
||||
- name: is_hs
|
||||
description: 是否沪深港通标的，N否 H沪股通 S深股通
|
||||
- name: act_name
|
||||
description: 实控人名称
|
||||
- name: act_ent_type
|
||||
description: 实控人企业性质
|
||||
- name: dt
|
||||
description: 最后修改时间
|
||||
unique_composite:
|
||||
- name: unique_exchange_ts_code
|
||||
- name: stg.tushare_trade_calendar
|
||||
description: >
|
||||
The source table for the trade calendar data from Tushare
|
||||
tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- exchange
|
||||
- cal_date
|
||||
columns:
|
||||
- exchange
|
||||
- ts_code
|
||||
- name: stg.tushare_daily
|
||||
description: "沪深股票交易日线数据"
|
||||
data_tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- ts_code
|
||||
- trade_date
|
||||
columns:
|
||||
- name: ts_code
|
||||
description: TS代码
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: trade_date
|
||||
description: 交易日期
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: open
|
||||
description: 开盘价
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: high
|
||||
description: 最高价
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: low
|
||||
description: 最低价
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: close
|
||||
description: 收盘价
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: pre_close
|
||||
description: 昨收价
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: change
|
||||
description: 涨跌额
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: pct_chg
|
||||
description: 涨跌幅
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: vol
|
||||
description: 成交量
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: amount
|
||||
description: 成交额
|
||||
- name: stg.tushare_daily_basic
|
||||
description: "沪深股票每日指标数据"
|
||||
data_tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- ts_code
|
||||
- trade_date
|
||||
- name: exchange
|
||||
description: "The exchange code"
|
||||
tests:
|
||||
- not_null
|
||||
- name: cal_date
|
||||
description: "The date"
|
||||
tests:
|
||||
- not_null
|
||||
- name: is_open
|
||||
description: "Whether the exchange is open"
|
||||
tests:
|
||||
- not_null
|
||||
- name: pretrade_date
|
||||
description: "The previous trading date"
|
||||
- name: stg.tushare_stock_basic
|
||||
columns:
|
||||
- name: ts_code
|
||||
description: TS代码
|
||||
tests:
|
||||
- not_null
|
||||
- name: symbol
|
||||
description: 股票代码
|
||||
- name: name
|
||||
description: 股票名称
|
||||
- name: area
|
||||
description: 所在地域
|
||||
- name: industry
|
||||
description: 所属行业
|
||||
- name: fullname
|
||||
description: 股票全称
|
||||
- name: enname
|
||||
description: 英文全称
|
||||
- name: cnspell
|
||||
description: 拼音缩写
|
||||
- name: market
|
||||
description: 市场类型 （主板/中小板/创业板）
|
||||
- name: exchange
|
||||
description: 交易所代码
|
||||
tests:
|
||||
- not_null
|
||||
- name: curr_type
|
||||
description: 交易货币
|
||||
- name: list_status
|
||||
description: 上市状态： L上市 D退市 P暂停上市
|
||||
- name: list_date
|
||||
description: 上市日期
|
||||
- name: delist_date
|
||||
description: 退市日期
|
||||
- name: is_hs
|
||||
description: 是否沪深港通标的，N否 H沪股通 S深股通
|
||||
- name: act_name
|
||||
description: 实控人名称
|
||||
- name: act_ent_type
|
||||
description: 实控人企业性质
|
||||
- name: dt
|
||||
description: 最后修改时间
|
||||
unique_composite:
|
||||
- name: unique_exchange_ts_code
|
||||
columns:
|
||||
- exchange
|
||||
- ts_code
|
||||
- name: stg.tushare_daily
|
||||
description: "沪深股票交易日线数据"
|
||||
tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- ts_code
|
||||
- trade_date
|
||||
columns:
|
||||
- name: ts_code
|
||||
description: TS代码
|
||||
tests:
|
||||
- not_null
|
||||
- name: trade_date
|
||||
description: 交易日期
|
||||
tests:
|
||||
- not_null
|
||||
- name: open
|
||||
description: 开盘价
|
||||
tests:
|
||||
- not_null
|
||||
- name: high
|
||||
description: 最高价
|
||||
tests:
|
||||
- not_null
|
||||
- name: low
|
||||
description: 最低价
|
||||
tests:
|
||||
- not_null
|
||||
- name: close
|
||||
description: 收盘价
|
||||
tests:
|
||||
- not_null
|
||||
- name: pre_close
|
||||
description: 昨收价
|
||||
tests:
|
||||
- not_null
|
||||
- name: change
|
||||
description: 涨跌额
|
||||
tests:
|
||||
- not_null
|
||||
- name: pct_chg
|
||||
description: 涨跌幅
|
||||
tests:
|
||||
- not_null
|
||||
- name: vol
|
||||
description: 成交量
|
||||
tests:
|
||||
- not_null
|
||||
- name: amount
|
||||
description: 成交额
|
||||
- name: stg.tushare_daily_basic
|
||||
description: "沪深股票每日指标数据"
|
||||
tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- ts_code
|
||||
- trade_date
|
||||
|
||||
columns:
|
||||
- name: ts_code
|
||||
description: "股票代码"
|
||||
data_type: String
|
||||
- name: trade_date
|
||||
description: "交易日期"
|
||||
data_type: String
|
||||
- name: close
|
||||
description: "当日收盘价"
|
||||
data_type: Nullable(Float32)
|
||||
- name: turnover_rate
|
||||
description: "换手率（%）"
|
||||
data_type: Nullable(Float32)
|
||||
- name: turnover_rate_f
|
||||
description: "换手率（自由流通股）"
|
||||
data_type: Nullable(Float32)
|
||||
- name: volume_ratio
|
||||
description: "量比"
|
||||
data_type: Nullable(Float32)
|
||||
- name: pe
|
||||
description: "市盈率（总市值/净利润， 亏损的PE为空）"
|
||||
data_type: Nullable(Float32)
|
||||
- name: pe_ttm
|
||||
description: "市盈率（TTM，亏损的PE为空）"
|
||||
data_type: Nullable(Float32)
|
||||
- name: pb
|
||||
description: "市净率（总市值/净资产）"
|
||||
data_type: Nullable(Float32)
|
||||
- name: ps
|
||||
description: "市销率"
|
||||
data_type: Nullable(Float32)
|
||||
- name: ps_ttm
|
||||
description: "市销率（TTM）"
|
||||
data_type: Nullable(Float32)
|
||||
- name: dv_ratio
|
||||
description: "股息率 （%）"
|
||||
data_type: Nullable(Float32)
|
||||
- name: dv_ttm
|
||||
description: "股息率（TTM）（%）"
|
||||
data_type: Nullable(Float32)
|
||||
- name: total_share
|
||||
description: "总股本 （万股）"
|
||||
data_type: Nullable(Float32)
|
||||
- name: float_share
|
||||
description: "流通股本 （万股）"
|
||||
data_type: Nullable(Float32)
|
||||
- name: free_share
|
||||
description: "自由流通股本 （万）"
|
||||
data_type: Nullable(Float32)
|
||||
- name: total_mv
|
||||
description: "总市值 （万元）"
|
||||
data_type: Nullable(Float32)
|
||||
- name: circ_mv
|
||||
description: "流通市值（万元）"
|
||||
data_type: Nullable(Float32)
|
||||
- name: stg.tushare_adj_factor
|
||||
description: "沪深股票复权因子"
|
||||
data_tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- ts_code
|
||||
- trade_date
|
||||
columns:
|
||||
- name: ts_code
|
||||
description: "股票代码"
|
||||
data_type: String
|
||||
- name: trade_date
|
||||
description: "交易日期"
|
||||
data_type: String
|
||||
- name: adj_factor
|
||||
description: "复权因子"
|
||||
data_type: Nullable(Decimal(16, 6))
|
||||
- name: dt
|
||||
description: "更新时间"
|
||||
data_type: DateTime
|
||||
- name: stg.tushare_minutes
|
||||
description: "沪深分钟级交易数据"
|
||||
data_tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- ts_code
|
||||
- trade_time
|
||||
config:
|
||||
where: "left(trade_time,7)>=concat(cast(year(today())-1 As String), '-01')"
|
||||
columns:
|
||||
- name: ts_code
|
||||
description: "The stock code."
|
||||
data_type: String
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: trade_time
|
||||
description: "The trading time."
|
||||
data_type: String
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: close
|
||||
description: "The closing price."
|
||||
data_type: Nullable(Float32)
|
||||
- name: open
|
||||
description: "The opening price."
|
||||
data_type: Nullable(Float32)
|
||||
- name: high
|
||||
description: "The highest price."
|
||||
data_type: Nullable(Float32)
|
||||
- name: low
|
||||
description: "The lowest price."
|
||||
data_type: Nullable(Float32)
|
||||
- name: vol
|
||||
description: "The volume of trades."
|
||||
data_type: Nullable(Float32)
|
||||
- name: amount
|
||||
description: "The amount of trades."
|
||||
data_type: Nullable(Float32)
|
||||
meta:
|
||||
engine: ReplacingMergeTree
|
||||
partition_by: "left(trade_time, 7)"
|
||||
order_by: "(ts_code, trade_time)"
|
||||
settings:
|
||||
index_granularity: 8192
|
||||
columns:
|
||||
- name: ts_code
|
||||
description: "股票代码"
|
||||
data_type: String
|
||||
- name: trade_date
|
||||
description: "交易日期"
|
||||
data_type: String
|
||||
- name: close
|
||||
description: "当日收盘价"
|
||||
data_type: Nullable(Float32)
|
||||
- name: turnover_rate
|
||||
description: "换手率（%）"
|
||||
data_type: Nullable(Float32)
|
||||
- name: turnover_rate_f
|
||||
description: "换手率（自由流通股）"
|
||||
data_type: Nullable(Float32)
|
||||
- name: volume_ratio
|
||||
description: "量比"
|
||||
data_type: Nullable(Float32)
|
||||
- name: pe
|
||||
description: "市盈率（总市值/净利润， 亏损的PE为空）"
|
||||
data_type: Nullable(Float32)
|
||||
- name: pe_ttm
|
||||
description: "市盈率（TTM，亏损的PE为空）"
|
||||
data_type: Nullable(Float32)
|
||||
- name: pb
|
||||
description: "市净率（总市值/净资产）"
|
||||
data_type: Nullable(Float32)
|
||||
- name: ps
|
||||
description: "市销率"
|
||||
data_type: Nullable(Float32)
|
||||
- name: ps_ttm
|
||||
description: "市销率（TTM）"
|
||||
data_type: Nullable(Float32)
|
||||
- name: dv_ratio
|
||||
description: "股息率 （%）"
|
||||
data_type: Nullable(Float32)
|
||||
- name: dv_ttm
|
||||
description: "股息率（TTM）（%）"
|
||||
data_type: Nullable(Float32)
|
||||
- name: total_share
|
||||
description: "总股本 （万股）"
|
||||
data_type: Nullable(Float32)
|
||||
- name: float_share
|
||||
description: "流通股本 （万股）"
|
||||
data_type: Nullable(Float32)
|
||||
- name: free_share
|
||||
description: "自由流通股本 （万）"
|
||||
data_type: Nullable(Float32)
|
||||
- name: total_mv
|
||||
description: "总市值 （万元）"
|
||||
data_type: Nullable(Float32)
|
||||
- name: circ_mv
|
||||
description: "流通市值（万元）"
|
||||
data_type: Nullable(Float32)
|
||||
- name: stg.tushare_adj_factor
|
||||
description: "沪深股票复权因子"
|
||||
tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- ts_code
|
||||
- trade_date
|
||||
columns:
|
||||
- name: ts_code
|
||||
description: "股票代码"
|
||||
data_type: String
|
||||
- name: trade_date
|
||||
description: "交易日期"
|
||||
data_type: String
|
||||
- name: adj_factor
|
||||
description: "复权因子"
|
||||
data_type: Nullable(Decimal(16, 6))
|
||||
- name: dt
|
||||
description: "更新日期"
|
||||
data_type: DateTime
|
||||
- name: stg.tushare_minutes
|
||||
description: "沪深分钟级交易数据"
|
||||
tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- ts_code
|
||||
- trade_time
|
||||
config:
|
||||
where: "left(trade_time,7)>=concat(cast(year(today())-1 As String), '-01')"
|
||||
columns:
|
||||
- name: ts_code
|
||||
description: "The stock code."
|
||||
data_type: String
|
||||
tests:
|
||||
- not_null
|
||||
- name: trade_time
|
||||
description: "The trading time."
|
||||
data_type: String
|
||||
tests:
|
||||
- not_null
|
||||
- name: close
|
||||
description: "The closing price."
|
||||
data_type: Nullable(Float32)
|
||||
- name: open
|
||||
description: "The opening price."
|
||||
data_type: Nullable(Float32)
|
||||
- name: high
|
||||
description: "The highest price."
|
||||
data_type: Nullable(Float32)
|
||||
- name: low
|
||||
description: "The lowest price."
|
||||
data_type: Nullable(Float32)
|
||||
- name: vol
|
||||
description: "The volume of trades."
|
||||
data_type: Nullable(Float32)
|
||||
- name: amount
|
||||
description: "The amount of trades."
|
||||
data_type: Nullable(Float32)
|
||||
meta:
|
||||
engine: ReplacingMergeTree
|
||||
partition_by: "left(trade_time, 7)"
|
||||
order_by: "(ts_code, trade_time)"
|
||||
settings:
|
||||
index_granularity: 8192
|
||||
|
||||
@ -1,19 +0,0 @@
|
||||
{{
|
||||
config(
|
||||
materialized='incremental',
|
||||
engine="MergeTree",
|
||||
order_by="(date_id, ts_code)",
|
||||
unique_key=['date_id', 'ts_code'],
|
||||
incremental_strategy='delete+insert',
|
||||
)
|
||||
}}
|
||||
|
||||
Select
|
||||
ts_code As ts_code,
|
||||
toYYYYMMDD(toDate(trade_date)) As date_id,
|
||||
adj_factor As adj_factor,
|
||||
dt As dt
|
||||
From {{ source('finance', 'stg.tushare_adj_factor') }} As StgAdjFactor
|
||||
{% if is_incremental() %}
|
||||
Where `trade_date` >= (Select formatDateTimeInJodaSyntax(YYYYMMDDToDate(max(`date_id`)), 'yyyy-MM-dd') From {{ this }})
|
||||
{% endif %}
|
||||
@ -1,35 +0,0 @@
|
||||
{{
|
||||
config(
|
||||
materialized='incremental',
|
||||
engine="MergeTree",
|
||||
order_by='(time_id, ts_code)',
|
||||
unique_key=['time_id', 'ts_code'],
|
||||
partition_by=['toYYYYMM(toDateTime(time_id))'],
|
||||
incremental_strategy='delete+insert',
|
||||
query_settings={
|
||||
"join_algorithm": "'full_sorting_merge'",
|
||||
"max_bytes_before_external_sort": "'1000M'",
|
||||
"max_bytes_before_external_group_by":"'1000M'",
|
||||
}
|
||||
)
|
||||
}}
|
||||
|
||||
Select
|
||||
StgMinutes.ts_code As ts_code,
|
||||
toYYYYMMDDhhmmss(toDateTime(`StgMinutes`.`trade_time`)) As time_id,
|
||||
StgMinutes.close As close,
|
||||
StgMinutes.open As open,
|
||||
StgMinutes.high As high,
|
||||
StgMinutes.low As low,
|
||||
StgMinutes.vol As vol,
|
||||
StgMinutes.amount As amount
|
||||
From {{ source('finance', 'stg.tushare_minutes') }} As StgMinutes
|
||||
where 1 = 1
|
||||
{% if is_incremental() %}
|
||||
And StgMinutes.trade_time >= (Select formatDateTimeInJodaSyntax(toStartOfDay(YYYYMMDDhhmmssToDateTime(max(time_id))), 'yyyy-MM-dd HH:mm:ss') From {{ this }})
|
||||
{% endif %}
|
||||
{% if 'dev' in target.name %}
|
||||
And StgMinutes.trade_time >= formatDateTimeInJodaSyntax(dateAdd(Month,-1,today()), 'yyyy-01-01 00:00:00')
|
||||
{% elif target.name == 'test' %}
|
||||
And StgMinutes.trade_time >= formatDateTimeInJodaSyntax(dateAdd(Year,-1,today()), 'yyyy-01-01 00:00:00')
|
||||
{% endif %}
|
||||
@ -1,54 +0,0 @@
|
||||
|
||||
version: 2
|
||||
|
||||
models:
|
||||
- name: ods.tushare_minutes
|
||||
description: "沪深股票交易分钟线数据"
|
||||
data_tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- ts_code
|
||||
- time_id
|
||||
columns:
|
||||
- name: ts_code
|
||||
description: TS代码
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: time_id
|
||||
description: 交易日期
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: close
|
||||
description: 收盘价
|
||||
- name: open
|
||||
description: 开盘价
|
||||
- name: high
|
||||
description: 最高价
|
||||
- name: low
|
||||
description: 最低价
|
||||
- name: vol
|
||||
description: 成交量
|
||||
- name: amount
|
||||
description: 成交额
|
||||
- name: dt
|
||||
description: 更新时间
|
||||
- name: ods.tushare_adj_factor
|
||||
description: "沪深股票复权因子"
|
||||
data_tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- ts_code
|
||||
- date_id
|
||||
columns:
|
||||
- name: ts_code
|
||||
description: TS代码
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: date_id
|
||||
description: 交易日期
|
||||
data_tests:
|
||||
- not_null
|
||||
- name: adj_factor
|
||||
description: 复权因子
|
||||
- name: dt
|
||||
description: 更新时间
|
||||
@ -15,7 +15,7 @@ finance_dbt:
|
||||
retries: 1
|
||||
compression: gzip
|
||||
connect_timeout: 10
|
||||
send_receive_timeout: 24000
|
||||
send_receive_timeout: 300
|
||||
cluster_mode: False
|
||||
use_lw_deletes: True
|
||||
check_exchange: True
|
||||
|
||||
@ -1 +1,2 @@
|
||||
dbt-clickhouse==1.8.0
|
||||
dbt==1.0.0.37.0
|
||||
dbt-clickhouse==1.7.3
|
||||
Loading…
Reference in New Issue
Block a user