init: create finance-dbt
Some checks failed
MJN/finance-dbt/pipeline/head There was a failure building this commit

This commit is contained in:
沈楠 2024-04-08 11:43:51 +08:00
commit 90e0f3852b
27 changed files with 1128 additions and 0 deletions

6
.env Normal file
View File

@ -0,0 +1,6 @@
FINANCE_DBT_TARGET=dev
FINANCE_CLICKHOUSE_HOST=localhost
FINANCE_CLICKHOUSE_HTTP_PORT=8123
FINANCE_CLICKHOUSE_DATABASE=finance_dev
FINANCE_CLICKHOUSE_USER=finance_dev
FINANCE_CLICKHOUSE_PASSWORD=<YOUR PASSWORD>

6
.gitignore vendored Normal file
View File

@ -0,0 +1,6 @@
.*
!.gitignore
!.env
/target/
/dbt_packages/
/logs/

10
CD.sh Normal file
View File

@ -0,0 +1,10 @@
if [ "dev" = "$BRANCH_NAME" ] || [ "test" = "$BRANCH_NAME" ] || [ "prod" = "$BRANCH_NAME" ]
then
export deployment=$BRANCH_NAME
export namespace=$BRANCH_NAME
else
export deployment=dev
export namespace=dev
fi
export image_tag=$IMAGE_TAG
. ./Deploy.sh

12
CI.sh Normal file
View File

@ -0,0 +1,12 @@
/kaniko/executor \
--context . \
--cache=true \
--cache-dir="${WORKSPACE}/kaniko-cache" \
--registry-mirror="${DOCKER_REGISTRY}" \
--destination "${DOCKER_REGISTRY}/${ORG}/${PROJECT}:${IMAGE_TAG}" \
--label org.opencontainers.image.branch=${BRANCH_NAME} \
--label org.opencontainers.image.build_tag=${BUILD_TAG} \
--label org.opencontainers.image.revision="${GIT_COMMIT}" \
--label org.opencontainers.image.version="${IMAGE_TAG}" \
--label org.opencontainers.image.url="${BUILD_URL}" \
--label org.opencontainers.image.source="${GIT_URL}"

35
DebugImage.sh Normal file
View File

@ -0,0 +1,35 @@
# build
tag=debug
docker_registry=docker-registry.mujiannan.com:5001
finance_dbt_image=${docker_registry}/mujiannan/finance-dbt:${tag}
docker build -t $finance_dbt_image . || exit 1
docker push $finance_dbt_image
# run
name=finance_dbt
docker stop $name || echo 'No container to stop...continue...'
docker rm $name || echo 'No container to remove...continue...'
docker run \
-d \
--env-file=.dev.env \
--name=$name \
$finance_dbt_image \
sleep 99d
# run doc-server
name=finance_dbt_doc
docker stop $name || echo 'No container to stop...continue...'
docker rm $name || echo 'No container to remove...continue...'
docker run \
-i \
-p 8080:8080 \
--env-file=.dev.env \
--name=$name \
--entrypoint=bash \
${finance_dbt_image} \
/app/doc-server.sh
## test kubernetes
export deployment=dev
export namespace=dev
export image_tag=debug
envsubst < deploy.yml | kubectl apply -f -

18
Deploy.sh Normal file
View File

@ -0,0 +1,18 @@
if [ -z "$deployment" ]; then
echo "deployment is not set"
exit 1
fi
if [ -z "$namespace" ]; then
echo "namespace is not set"
exit 1
fi
if [ -z "$image_tag" ]; then
echo "image_tag is not set"
exit 1
fi
if { [ "$deployment" = "dev" ] || [ "$deployment" = "test" ] }; then
ingress_host=finance-dbt-doc.${deployment}.mujiannan.com
else
ingress_host=finance-dbt-doc.mujiannan.com
fi
envsubst < Deploy.yml | kubectl apply -f -

65
Deploy.yml Normal file
View File

@ -0,0 +1,65 @@
apiVersion: v1
kind: Service
metadata:
name: finance-dbt-doc
namespace: ${namespace}
spec:
selector:
app: finance-dbt-doc
ports:
- port: 8080
targetPort: 8080
protocol: TCP
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: finance-dbt-doc
namespace: ${namespace}
spec:
replicas: 1
selector:
matchLabels:
app: finance-dbt-doc
template:
metadata:
labels:
app: finance-dbt-doc
spec:
containers:
- name: finance-dbt-doc
image: docker-registry.mujiannan.com:5001/mujiannan/finance-dbt:${image_tag}
ports:
- containerPort: 8080
command: ["bash", "/app/doc-server.sh", ""]
resources:
requests:
cpu: 10m
memory: 10Mi
limits:
cpu: 1000m
memory: 200Mi
envFrom:
- secretRef:
name: finance-dbt-doc-envs
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: finance-dbt-doc
namespace: ${namespace}
annotations:
nginx.ingress.kubernetes.io/rewrite-target: /$1
spec:
ingressClassName: nginx
rules:
- host: ${ingress_host}
http:
paths:
- pathType: Prefix
path: /(.*)
backend:
service:
name: finance-dbt-doc
port:
number: 8080

5
DeploySecrets.sh Normal file
View File

@ -0,0 +1,5 @@
# Pul your env file on local machine
kubectl create secret generic finance-dbt-doc-envs --from-env-file=./.dev.env --namespace=dev
kubectl create secret generic finance-dbt-doc-envs --from-env-file=./.test.env --namespace=test
kubectl create secret generic finance-dbt-doc-envs --from-env-file=./.prod.env --namespace=prod

14
Dockerfile Normal file
View File

@ -0,0 +1,14 @@
FROM python:3.10.14-slim
USER 1001
WORKDIR /app
### Pip
USER root
COPY ./requirements.txt /app/
RUN pip3 config set global.index-url https://nexus.mujiannan.com:5001/repository/pypiserver/simple && \
pip install -r requirements.txt
USER 1001
## Copy files
COPY --chown=1001 . /app
## Install packges/dependencies
### Dbt
RUN dbt deps

73
Jenkinsfile vendored Normal file
View File

@ -0,0 +1,73 @@
/* groovylint-disable NestedBlockDepth */
// Description: Jenkinsfile for finance_dbt
pipeline {
agent {
kubernetes {
inheritFrom 'kaniko'
}
}
environment {
DESTINATION_MACHINE = 'shennan@physical-ubuntu-server'
ORG = 'mujiannan'
PROJECT = 'finance-dbt'
DOCKER_REGISTRY = 'docker-registry.mujiannan.com:5001'
SMS_URL = 'https://nas.mujiannan.com:5001/webapi/entry.cgi?api=SYNO.Chat.External&method=incoming&version=2&token=%22XO8yFQyi66SStEPY7AALimGbNjMsZE85i3m0UlF0siIQtyn1deqSomp0CUheOwlE%22' // NOSONAR
}
stages {
stage('Pre Check') {
steps {
script {
// Retrieve the latest commit message
def String commitMessage = sh(returnStdout: 'true', script: 'git log -1 --pretty=%B').trim()
echo "Commit message: ${commitMessage}"
// Set a flag based on the commit message
if (commitMessage.contains('[SKIP CI]')) {
env.SKIP_CI = 'true'
}
// Set a flag based on the commit message
if (
commitMessage.contains('[SKIP CD]') || env.SKIP_CI == 'true'
) {
env.SKIP_CD = 'true'
}
}
script {
// Set the image tag, GIT_TAG is preferred, if not available, use the commit hash abbreviated
env.IMAGE_TAG = sh(returnStdout: 'true', script: 'git describe --tags --always').trim()
}
}
}
stage('Build') {
when {
not {
expression {
return env.SKIP_CI == 'true'
}
}
}
steps {
container('kaniko') {
// Build the Docker image
sh '. ./CI.sh'
}
}
}
stage('Deploy') {
when {
not {
expression {
return env.SKIP_CD == 'true'
}
}
}
steps {
withKubeConfig([namespace: "${BRANCH_NAME}"]) {
container('kubectl') {
sh '. ./CD.sh'
}
}
}
}
}
}

33
README.md Normal file
View File

@ -0,0 +1,33 @@
# Welcome to finance dbt project!
### Prepare
```bash
cp env.sh env.dev.sh
```
Change configurations in .env.dev.sh
```bash
source env.dev.sh
```
### Run dbt
Try running the following commands:
- dbt deps
- dbt run
- dbt test
### Run in docker
1. Run pipeline
```bash
docker run <THIS_IMAGE> <DBT-COMMAND>
```
2. As document-server
```bash
docker run -p <PORT>:80 <THIS_IMAGE> sleep 99d
```
### Resources:
- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction)
- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers
- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support
- Find [dbt events](https://events.getdbt.com) near you
- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices

37
dbt_project.yml Normal file
View File

@ -0,0 +1,37 @@
# Name your project! Project names should contain only lowercase characters
# and underscores. A good package name should reflect your organization's
# name or the intended use of these models
name: 'finance_dbt'
version: '1.0.0'
config-version: 2
# This setting configures which "profile" dbt uses for this project.
profile: 'finance_dbt'
# These configurations specify where dbt should look for different types of files.
# The `model-paths` config, for example, states that models in this project can be
# found in the "models/" directory. You probably won't need to change these!
model-paths: ["models"]
analysis-paths: ["analyses"]
test-paths: ["tests"]
seed-paths: ["seeds"]
macro-paths: ["macros"]
snapshot-paths: ["snapshots"]
clean-targets: # directories to be removed by `dbt clean`
- "target"
- "dbt_packages"
# Configuring models
# Full documentation: https://docs.getdbt.com/docs/configuring-models
# In this example config, we tell dbt to build all models in the example/
# directory as views. These settings can be overridden in the individual model
# files using the `{{ config(...) }}` macro.
models:
finance_dbt:
# Config indicated by + and applies to all files under models/example/
dw:
+materialized: table

2
doc-server.sh Normal file
View File

@ -0,0 +1,2 @@
dbt docs generate
dbt docs serve

View File

@ -0,0 +1,3 @@
{% macro optimize_table_final(table_name) %}
Optimize Table {{ table_name }} Final
{% endmacro %}

30
models/dw/dw.dim_date.sql Normal file
View File

@ -0,0 +1,30 @@
/*
*/
{{
config(
materialized='table',
engine='MergeTree',
order_by='date_id'
)
}}
SELECT
toUInt32(year(date) * 10000 + month(date) * 100 + day(date)) AS date_id,
date AS full_date,
toYear(date) AS year,
toUInt8(quarter(date)) AS quarter,
toUInt8(month(date)) AS month,
toUInt8(toDayOfMonth(date)) AS day,
toUInt8(week(date)) AS week_num_of_year,
toUInt8(toDayOfWeek(date)) AS day_num_of_week
FROM (
SELECT
arrayJoin(
arrayMap(
x -> toDate('1990-01-01') + x,
range(toUInt32(dateDiff('day', toDate('1990-01-01'), toDate('2031-01-01'))))
)
) AS date
)
ORDER BY date_id

View File

@ -0,0 +1,20 @@
/*
*/
{{ config(materialized='view') }}
with source_data as (
select 0 As exchange_id
, 'SSE' As ts_exchange_code
, '上海证券交易所' As exchange_name
Union All
select 1 As exchange_id
, 'SZSE' As ts_exchange_code
, '深圳证券交易所' As exchange_name
)
select *
from source_data

View File

@ -0,0 +1,17 @@
/*
*/
{{
config(
materialized='table',
engine="MergeTree",
order_by='(exchange_id, date_id)',
)
}}
Select `Exchange`.`exchange_id` As `exchange_id`
, Cast(`StgCalendar`.`cal_date` As Int32) As `date_id`
, Cast(`StgCalendar`.`is_open` As Int8) As `is_open`
From {{ source('finance', 'stg.tushare_trade_calendar') }} As `StgCalendar` Final
All Inner Join {{ ref('dw.dim_exchange') }} As `Exchange` Final
On `StgCalendar`.`exchange` = `Exchange`.`ts_exchange_code`

View File

@ -0,0 +1,45 @@
/*
*/
{{
config(
materialized='table',
engine='MergeTree',
order_by='ts_code'
)
}}
SELECT
StgStock.ts_code As ts_code
, DimExchange.exchange_id As exchange_id
, arrayElement(splitByChar('.', StgStock.`ts_code`),1) As code
, StgStock.name As name
, StgStock.fullname As full_name
, StgStock.enname As name_en
, StgStock.cnspell As cn_spell
, StgStock.area As area
, StgStock.industry As industry
, StgStock.market As market
, StgStock.curr_type As currency
, ListDate.full_date As list_date
, Case When StgStock.delist_date Is Null Or StgStock.delist_date = '' Then Null Else DelistDate.full_date End As delist_date
, Case
When StgStock.list_status = 'L'
Then 1
Else 0
End As is_active
, Case
When is_hs In ('H', 'S')
Then 1
Else 0
End As is_hongkong_connect
, act_name As actual_controller
, act_ent_type As actual_controller_enterprise_type
, now() As dt
From {{ source('finance', 'stg.tushare_stock_basic') }} As StgStock Final
Left Join {{ ref("dw.dim_exchange") }} As DimExchange
On StgStock.exchange = DimExchange.ts_exchange_code
Left Join {{ ref("dw.dim_date") }} As ListDate
On StgStock.list_date = Cast(ListDate.date_id As String)
Left Join {{ ref("dw.dim_date") }} As DelistDate
On StgStock.delist_date = Cast(DelistDate.date_id As String)

24
models/dw/dw.dim_time.sql Normal file
View File

@ -0,0 +1,24 @@
{{
config(
materialized='table',
engine='MergeTree',
order_by='time_id'
)
}}
Select cast(formatDateTime(`full_time`, '%Y%m%d%H%i') As Int64) As time_id
, `full_time`
, `date_id`
, hour(`full_time`) As `hour`
, minute(`full_time`) As `minute`
From(
SELECT
arrayJoin(
arrayMap(
x -> toStartOfMinute(toDateTime(toDate(`full_date`)) + x * 60),
range(0, 24 * 60)
)
) AS `full_time`
, `date_id`
From {{ ref("dw.dim_date") }}
Where `date_id` >= 20090101
) As T

View File

@ -0,0 +1,49 @@
{{
config(
materialized='incremental',
engine='MergeTree',
order_by='ts_code, date_id',
unique_key=['ts_code', 'date_id'],
incremental_strategy='delete+insert',
)
}}
Select
StgDaily.ts_code As ts_code,
DimDate.date_id As date_id,
StgDaily.open As open,
StgDaily.high As high,
StgDaily.low As low,
StgDaily.close As close,
StgDaily.pre_close As pre_close,
StgDaily.change As change,
StgDaily.pct_chg As pct_chg,
StgDaily.vol As vol,
StgDaily.amount As amount,
StgAdjFactor.adj_factor As adj_factor,
StgDailyBasic.turnover_rate As turnover_rate,
StgDailyBasic.turnover_rate_f As turnover_rate_f,
StgDailyBasic.volume_ratio As volume_ratio,
StgDailyBasic.pe As pe,
StgDailyBasic.pe_ttm As pe_ttm,
StgDailyBasic.pb As pb,
StgDailyBasic.ps As ps,
StgDailyBasic.ps_ttm As ps_ttm,
StgDailyBasic.dv_ratio As dv_ratio,
StgDailyBasic.dv_ttm As dv_ttm,
StgDailyBasic.total_share As total_share,
StgDailyBasic.float_share As float_share,
StgDailyBasic.free_share As free_share,
StgDailyBasic.total_mv As total_mv,
StgDailyBasic.circ_mv As circ_mv
From {{ source('finance', 'stg.tushare_daily') }} As StgDaily Final
Inner Join {{ ref('dw.dim_date') }} As DimDate
On StgDaily.trade_date = Cast(DimDate.date_id As String)
Any Left Join {{ source('finance', 'stg.tushare_daily_basic') }} As StgDailyBasic
On StgDaily.ts_code = StgDailyBasic.ts_code
And StgDaily.trade_date = StgDailyBasic.trade_date
Any Left Join {{ source('finance', 'stg.tushare_adj_factor') }} As StgAdjFactor
On StgDaily.ts_code = StgAdjFactor.ts_code
And StgDaily.trade_date = StgAdjFactor.trade_date
{% if is_incremental() %}
Where StgDaily.trade_date >= (Select cast(max(date_id) As String) From {{ this }})
{% endif %}

View File

@ -0,0 +1,34 @@
{{
config(
materialized='materialized_view',
engine='MergeTree',
order_by='ts_code, time_id',
unique_key=['ts_code', 'time_id'],
partition_by=['cast(floor(time_id / 1000000) As Int32)'],
query_settings={"join_algorithm": "'partial_merge'"}
)
}}
Select
StgMinutes.ts_code As ts_code,
DimTime.time_id As time_id,
StgMinutes.close As close,
StgMinutes.open As open,
StgMinutes.high As high,
StgMinutes.low As low,
StgMinutes.vol As vol,
StgMinutes.amount As amount,
StgAdjFactor.adj_factor As adj_factor,
now() As dt
From {{ source('finance', 'stg.tushare_minutes') }} As StgMinutes
Inner Join {{ ref('dw.dim_time') }} As DimTime
On StgMinutes.trade_time = formatDateTimeInJodaSyntax(full_time, 'yyyy-MM-dd HH:mm:ss')
Any Left Join {{ source('finance', 'stg.tushare_adj_factor') }} As StgAdjFactor
On StgMinutes.ts_code = StgAdjFactor.ts_code
And StgAdjFactor.trade_date = Cast(DimTime.date_id As String)
where 1 = 1
{% if 'dev' in target.name %}
And StgMinutes.trade_time >= formatDateTimeInJodaSyntax(today(), 'yyyy-01-01 00:00:00')
{% elif target.name == 'test' %}
And StgMinutes.trade_time >= formatDateTimeInJodaSyntax(dateAdd(Year,-1,today()), 'yyyy-01-01 00:00:00')
{% endif %}

292
models/dw/schema.yml Normal file
View File

@ -0,0 +1,292 @@
version: 2
models:
- name: dw.dim_exchange
description: "交易所"
columns:
- name: exchange_id
description: "The primary key for this table"
tests:
- unique
- not_null
- name: ts_exchange_code
description: "交易所代码"
tests:
- not_null
- unique
- name: exchange_name
description: "交易所名称"
tests:
- not_null
- unique
- name: dw.dim_date
description: "日期"
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- year
- month
- day
columns:
- name: date_id
description: "日期键"
tests:
- unique
- not_null
- name: full_date
description: "日期"
tests:
- not_null
- unique
- name: year
description: "年"
tests:
- not_null
- name: quarter
description: "季度"
tests:
- not_null
- name: month
description: "月"
tests:
- not_null
- name: day
description: "日"
tests:
- not_null
- name: week_num_of_year
description: "年中第几周"
tests:
- not_null
- name: day_num_of_week
description: "星期"
tests:
- not_null
- name: dw.dim_time
description: "时间"
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- date_id
- hour
- minute
columns:
- name: time_id
description: "时间键"
tests:
- unique
- not_null
- name: full_time
description: "时间"
tests:
- not_null
- unique
- name: date_id
description: "日期键"
tests:
- not_null
- name: hour
description: "小时"
tests:
- not_null
- name: minute
description: "分钟"
tests:
- not_null
- name: dw.dim_hs_calendar
description: "沪深交易日历"
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- exchange_id
- date_id
columns:
- name: exchange_id
description: "The primary key for this table"
tests:
- not_null
- name: date_id
description: "交易日期键"
tests:
- not_null
- name: is_open
description: "是否交易"
tests:
- not_null
- name: dw.dim_stock
description: "股票"
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- exchange_id
- code
columns:
- name: ts_code
description: "tushare 股票编码"
tests:
- unique
- not_null
- name: exchange_id
description: "交易所键"
tests:
- not_null
- name: code
description: "股票代码"
tests:
- not_null
- name: name
description: "股票名称"
tests:
- not_null
- unique
- name: full_name
description: "股票全称"
tests:
- not_null
- unique
- name: name_en
description: "股票英文名称"
tests:
- not_null
- name: cn_spell
description: "拼音缩写"
tests:
- not_null
- name: area
description: "地区"
- name: industry
description: "行业"
- name: market
description: "市场类型 (主板/中小板/创业板)"
- name: currency
description: "交易货币"
- name: list_date
description: "上市日期"
- name: delist_date
description: "退市日期"
- name: is_active
description: "当前是否在市"
tests:
- not_null
- name: is_hongkong_connect
description: "是否沪深通标的"
tests:
- not_null
- name: actual_controller
description: "实际控制人"
- name: actual_controller_enterprise_type
description: "实际控制人企业类型"
- name: dt
description: "更新时间"
- name: dw.fact_stock_daily
description: "股票日数据"
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- ts_code
- date_id
columns:
- name: ts_code
description: "股票键"
tests:
- not_null
- name: date_id
description: "日期键"
tests:
- not_null
- name: open
description: "开盘价"
- name: high
description: "最高价"
- name: low
description: "最低价"
- name: close
description: "收盘价"
- name: pre_close
description: "昨收价"
- name: change
description: "涨跌额"
- name: pct_chg
description: "涨跌幅"
- name: vol
description: "成交量"
- name: amount
description: "成交额"
- name: adj_factor
description: "复权因子"
- name: turnover_rate
description: "换手率"
- name: turnover_rate_free
description: "换手率(自由流通股)"
- name: volume_ratio
description: "量比"
- name: pe
description: "市盈率(总市值/净利润, 亏损的PE为空"
- name: pe_ttm
description: "市盈率TTM"
- name: pb
description: "市净率(总市值/净资产)"
- name: ps
description: "市销率"
- name: ps_ttm
description: "市销率TTM"
- name: dv_ratio
description: "股息率"
- name: dv_ttm
description: "股息率TTM"
- name: total_share
description: "总股本(万股)"
- name: float_share
description: "流通股本(万股)"
- name: free_share
description: "自由流通股本(万)"
- name: total_mv
description: "总市值(万元)"
- name: circ_mv
description: "流通市值(万元)"
- name: dw.fact_stock_minute
description: "股票分钟数据"
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- ts_code
- time_id
columns:
- name: ts_code
description: "股票键"
data_type: "String"
tests:
- not_null
- name: time_id
description: "时间键"
data_type: "Int64"
tests:
- not_null
- name: open
description: "开盘价"
data_type: "Nullable(Decimal(16,2))"
- name: high
description: "最高价"
data_type: "Nullable(Decimal(16,2))"
- name: low
description: "最低价"
data_type: "Nullable(Decimal(16,2))"
- name: close
description: "收盘价"
data_type: "Nullable(Decimal(16,2))"
- name: vol
description: "成交量"
data_type: "Nullable(Int32)"
- name: amount
description: "成交额"
data_type: "Nullable(Decimal(18,2))"
- name: adj_factor
description: "复权因子"
data_type: "Decimal(16,6)"
tests:
- not_null
- name: dt
description: "时间"
data_type: "DateTime"

253
models/finance_source.yml Normal file
View File

@ -0,0 +1,253 @@
version: 2
sources:
- name: finance
schema: '{{ env_var("FINANCE_CLICKHOUSE_DATABASE") }}'
description: "The source for financial data"
tables:
- name: stg.tushare_trade_calendar
description: >
The source table for the trade calendar data from Tushare
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- exchange
- cal_date
columns:
- name: exchange
description: "The exchange code"
tests:
- not_null
- name: cal_date
description: "The date"
tests:
- not_null
- name: is_open
description: "Whether the exchange is open"
tests:
- not_null
- name: pretrade_date
description: "The previous trading date"
- name: stg.tushare_stock_basic
columns:
- name: ts_code
description: TS代码
tests:
- not_null
- name: symbol
description: 股票代码
- name: name
description: 股票名称
- name: area
description: 所在地域
- name: industry
description: 所属行业
- name: fullname
description: 股票全称
- name: enname
description: 英文全称
- name: cnspell
description: 拼音缩写
- name: market
description: 市场类型 (主板/中小板/创业板)
- name: exchange
description: 交易所代码
tests:
- not_null
- name: curr_type
description: 交易货币
- name: list_status
description: 上市状态: L上市 D退市 P暂停上市
- name: list_date
description: 上市日期
- name: delist_date
description: 退市日期
- name: is_hs
description: 是否沪深港通标的N否 H沪股通 S深股通
- name: act_name
description: 实控人名称
- name: act_ent_type
description: 实控人企业性质
- name: dt
description: 最后修改时间
unique_composite:
- name: unique_exchange_ts_code
columns:
- exchange
- ts_code
- name: stg.tushare_daily
description: "沪深股票交易日线数据"
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- ts_code
- trade_date
columns:
- name: ts_code
description: TS代码
tests:
- not_null
- name: trade_date
description: 交易日期
tests:
- not_null
- name: open
description: 开盘价
tests:
- not_null
- name: high
description: 最高价
tests:
- not_null
- name: low
description: 最低价
tests:
- not_null
- name: close
description: 收盘价
tests:
- not_null
- name: pre_close
description: 昨收价
tests:
- not_null
- name: change
description: 涨跌额
tests:
- not_null
- name: pct_chg
description: 涨跌幅
tests:
- not_null
- name: vol
description: 成交量
tests:
- not_null
- name: amount
description: 成交额
- name: stg.tushare_daily_basic
description: "沪深股票每日指标数据"
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- ts_code
- trade_date
columns:
- name: ts_code
description: "股票代码"
data_type: String
- name: trade_date
description: "交易日期"
data_type: String
- name: close
description: "当日收盘价"
data_type: Nullable(Float32)
- name: turnover_rate
description: "换手率(%"
data_type: Nullable(Float32)
- name: turnover_rate_f
description: "换手率(自由流通股)"
data_type: Nullable(Float32)
- name: volume_ratio
description: "量比"
data_type: Nullable(Float32)
- name: pe
description: "市盈率(总市值/净利润, 亏损的PE为空"
data_type: Nullable(Float32)
- name: pe_ttm
description: "市盈率TTM亏损的PE为空"
data_type: Nullable(Float32)
- name: pb
description: "市净率(总市值/净资产)"
data_type: Nullable(Float32)
- name: ps
description: "市销率"
data_type: Nullable(Float32)
- name: ps_ttm
description: "市销率TTM"
data_type: Nullable(Float32)
- name: dv_ratio
description: "股息率 %"
data_type: Nullable(Float32)
- name: dv_ttm
description: "股息率TTM%"
data_type: Nullable(Float32)
- name: total_share
description: "总股本 (万股)"
data_type: Nullable(Float32)
- name: float_share
description: "流通股本 (万股)"
data_type: Nullable(Float32)
- name: free_share
description: "自由流通股本 (万)"
data_type: Nullable(Float32)
- name: total_mv
description: "总市值 (万元)"
data_type: Nullable(Float32)
- name: circ_mv
description: "流通市值(万元)"
data_type: Nullable(Float32)
- name: stg.tushare_adj_factor
description: "沪深股票复权因子"
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- ts_code
- trade_date
columns:
- name: ts_code
description: "股票代码"
data_type: String
- name: trade_date
description: "交易日期"
data_type: String
- name: adj_factor
description: "复权因子"
data_type: Nullable(Decimal(16, 6))
- name: dt
description: "更新日期"
data_type: DateTime
- name: stg.tushare_minutes
description: "沪深分钟级交易数据"
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- ts_code
- trade_time
config:
where: "left(trade_time,7)>=concat(cast(year(today())-1 As String), '-01')"
columns:
- name: ts_code
description: "The stock code."
data_type: String
tests:
- not_null
- name: trade_time
description: "The trading time."
data_type: String
tests:
- not_null
- name: close
description: "The closing price."
data_type: Nullable(Float32)
- name: open
description: "The opening price."
data_type: Nullable(Float32)
- name: high
description: "The highest price."
data_type: Nullable(Float32)
- name: low
description: "The lowest price."
data_type: Nullable(Float32)
- name: vol
description: "The volume of trades."
data_type: Nullable(Float32)
- name: amount
description: "The amount of trades."
data_type: Nullable(Float32)
meta:
engine: ReplacingMergeTree
partition_by: "left(trade_time, 7)"
order_by: "(ts_code, trade_time)"
settings:
index_granularity: 8192

4
package-lock.yml Normal file
View File

@ -0,0 +1,4 @@
packages:
- package: dbt-labs/dbt_utils
version: 1.1.1
sha1_hash: a158c48c59c2bb7d729d2a4e215aabe5bb4f3353

3
packages.yml Normal file
View File

@ -0,0 +1,3 @@
packages:
- package: dbt-labs/dbt_utils
version: 1.1.1

36
profiles.yml Normal file
View File

@ -0,0 +1,36 @@
finance_dbt:
target: "{{ env_var('FINANCE_DBT_TARGET') }}"
outputs:
.template: &default_template # 定义锚点
type: clickhouse
schema: "{{ env_var('FINANCE_CLICKHOUSE_DATABASE') }}"
# optional
driver: http
host: "{{ env_var('FINANCE_CLICKHOUSE_HOST') }}"
port: "{{ env_var('FINANCE_CLICKHOUSE_HTTP_PORT') | as_number }}"
user: "{{ env_var('FINANCE_CLICKHOUSE_USER') }}"
password: "{{ env_var('FINANCE_CLICKHOUSE_PASSWORD') }}"
verify: True
secure: False
retries: 1
compression: gzip
connect_timeout: 10
send_receive_timeout: 300
cluster_mode: False
use_lw_deletes: True
check_exchange: True
local_suffix: _local
allow_automatic_deduplication: True
custom_settings: {}
# Native (clickhouse-driver) connection settings
sync_request_timeout: 5
compress_block_size: 1048576
dev:
<<: *default_template # 引用锚点
test:
<<: *default_template # 引用锚点
prod:
<<: *default_template # 引用锚点
local_dev:
<<: *default_template # 引用锚点

2
requirements.txt Normal file
View File

@ -0,0 +1,2 @@
dbt==1.0.0.37.0
dbt-clickhouse==1.7.3