init: create finance-dbt
Some checks failed
MJN/finance-dbt/pipeline/head There was a failure building this commit
Some checks failed
MJN/finance-dbt/pipeline/head There was a failure building this commit
This commit is contained in:
commit
0734f50550
6
.env
Normal file
6
.env
Normal file
@ -0,0 +1,6 @@
|
||||
FINANCE_DBT_TARGET=dev
|
||||
FINANCE_CLICKHOUSE_HOST=localhost
|
||||
FINANCE_CLICKHOUSE_HTTP_PORT=8123
|
||||
FINANCE_CLICKHOUSE_DATABASE=finance_dev
|
||||
FINANCE_CLICKHOUSE_USER=finance_dev
|
||||
FINANCE_CLICKHOUSE_PASSWORD=<YOUR PASSWORD>
|
6
.gitignore
vendored
Normal file
6
.gitignore
vendored
Normal file
@ -0,0 +1,6 @@
|
||||
.*
|
||||
!.gitignore
|
||||
!.env
|
||||
/target/
|
||||
/dbt_packages/
|
||||
/logs/
|
9
CD.sh
Normal file
9
CD.sh
Normal file
@ -0,0 +1,9 @@
|
||||
if [ "dev" = "$BRANCH_NAME" ] || [ "test" = "$BRANCH_NAME" ] || [ "prod" = "$BRANCH_NAME" ]; then
|
||||
export deployment=$BRANCH_NAME
|
||||
export namespace=$BRANCH_NAME
|
||||
else
|
||||
export deployment=dev
|
||||
export namespace=dev
|
||||
fi
|
||||
export image_tag=$IMAGE_TAG
|
||||
. ./Deploy.sh
|
12
CI.sh
Normal file
12
CI.sh
Normal file
@ -0,0 +1,12 @@
|
||||
/kaniko/executor \
|
||||
--context . \
|
||||
--cache=true \
|
||||
--cache-dir="${WORKSPACE}/kaniko-cache" \
|
||||
--registry-mirror="${DOCKER_REGISTRY}" \
|
||||
--destination "${DOCKER_REGISTRY}/${ORG}/${PROJECT}:${IMAGE_TAG}" \
|
||||
--label org.opencontainers.image.branch=${BRANCH_NAME} \
|
||||
--label org.opencontainers.image.build_tag=${BUILD_TAG} \
|
||||
--label org.opencontainers.image.revision="${GIT_COMMIT}" \
|
||||
--label org.opencontainers.image.version="${IMAGE_TAG}" \
|
||||
--label org.opencontainers.image.url="${BUILD_URL}" \
|
||||
--label org.opencontainers.image.source="${GIT_URL}"
|
35
DebugImage.sh
Normal file
35
DebugImage.sh
Normal file
@ -0,0 +1,35 @@
|
||||
# build
|
||||
tag=debug
|
||||
docker_registry=docker-registry.mujiannan.com:5001
|
||||
finance_dbt_image=${docker_registry}/mujiannan/finance-dbt:${tag}
|
||||
docker build -t $finance_dbt_image . || exit 1
|
||||
docker push $finance_dbt_image
|
||||
# run
|
||||
name=finance_dbt
|
||||
docker stop $name || echo 'No container to stop...continue...'
|
||||
docker rm $name || echo 'No container to remove...continue...'
|
||||
docker run \
|
||||
-d \
|
||||
--env-file=.dev.env \
|
||||
--name=$name \
|
||||
$finance_dbt_image \
|
||||
sleep 99d
|
||||
|
||||
# run doc-server
|
||||
name=finance_dbt_doc
|
||||
docker stop $name || echo 'No container to stop...continue...'
|
||||
docker rm $name || echo 'No container to remove...continue...'
|
||||
docker run \
|
||||
-i \
|
||||
-p 8080:8080 \
|
||||
--env-file=.dev.env \
|
||||
--name=$name \
|
||||
--entrypoint=bash \
|
||||
${finance_dbt_image} \
|
||||
/app/doc-server.sh
|
||||
|
||||
## test kubernetes
|
||||
export deployment=dev
|
||||
export namespace=dev
|
||||
export image_tag=debug
|
||||
envsubst < deploy.yml | kubectl apply -f -
|
18
Deploy.sh
Normal file
18
Deploy.sh
Normal file
@ -0,0 +1,18 @@
|
||||
if [ -z "$deployment" ]; then
|
||||
echo "deployment is not set"
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$namespace" ]; then
|
||||
echo "namespace is not set"
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$image_tag" ]; then
|
||||
echo "image_tag is not set"
|
||||
exit 1
|
||||
fi
|
||||
if { [ "$deployment" = "dev" ] || [ "$deployment" = "test" ] }; then
|
||||
ingress_host=finance-dbt-doc.${deployment}.mujiannan.com
|
||||
else
|
||||
ingress_host=finance-dbt-doc.mujiannan.com
|
||||
fi
|
||||
envsubst < Deploy.yml | kubectl apply -f -
|
65
Deploy.yml
Normal file
65
Deploy.yml
Normal file
@ -0,0 +1,65 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: finance-dbt-doc
|
||||
namespace: ${namespace}
|
||||
spec:
|
||||
selector:
|
||||
app: finance-dbt-doc
|
||||
ports:
|
||||
- port: 8080
|
||||
targetPort: 8080
|
||||
protocol: TCP
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: finance-dbt-doc
|
||||
namespace: ${namespace}
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: finance-dbt-doc
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: finance-dbt-doc
|
||||
spec:
|
||||
containers:
|
||||
- name: finance-dbt-doc
|
||||
image: docker-registry.mujiannan.com:5001/mujiannan/finance-dbt:${image_tag}
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
command: ["bash", "/app/doc-server.sh", ""]
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 10Mi
|
||||
limits:
|
||||
cpu: 1000m
|
||||
memory: 200Mi
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: finance-dbt-doc-envs
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: finance-dbt-doc
|
||||
namespace: ${namespace}
|
||||
annotations:
|
||||
nginx.ingress.kubernetes.io/rewrite-target: /$1
|
||||
spec:
|
||||
ingressClassName: nginx
|
||||
rules:
|
||||
- host: ${ingress_host}
|
||||
http:
|
||||
paths:
|
||||
- pathType: Prefix
|
||||
path: /(.*)
|
||||
backend:
|
||||
service:
|
||||
name: finance-dbt-doc
|
||||
port:
|
||||
number: 8080
|
5
DeploySecrets.sh
Normal file
5
DeploySecrets.sh
Normal file
@ -0,0 +1,5 @@
|
||||
|
||||
# Pul your env file on local machine
|
||||
kubectl create secret generic finance-dbt-doc-envs --from-env-file=./.dev.env --namespace=dev
|
||||
kubectl create secret generic finance-dbt-doc-envs --from-env-file=./.test.env --namespace=test
|
||||
kubectl create secret generic finance-dbt-doc-envs --from-env-file=./.prod.env --namespace=prod
|
14
Dockerfile
Normal file
14
Dockerfile
Normal file
@ -0,0 +1,14 @@
|
||||
FROM python:3.10.14-slim
|
||||
USER 1001
|
||||
WORKDIR /app
|
||||
### Pip
|
||||
USER root
|
||||
COPY ./requirements.txt /app/
|
||||
RUN pip3 config set global.index-url https://nexus.mujiannan.com:5001/repository/pypiserver/simple && \
|
||||
pip install -r requirements.txt
|
||||
USER 1001
|
||||
## Copy files
|
||||
COPY --chown=1001 . /app
|
||||
## Install packges/dependencies
|
||||
### Dbt
|
||||
RUN dbt deps
|
73
Jenkinsfile
vendored
Normal file
73
Jenkinsfile
vendored
Normal file
@ -0,0 +1,73 @@
|
||||
/* groovylint-disable NestedBlockDepth */
|
||||
// Description: Jenkinsfile for finance_dbt
|
||||
|
||||
pipeline {
|
||||
agent {
|
||||
kubernetes {
|
||||
inheritFrom 'kaniko'
|
||||
}
|
||||
}
|
||||
environment {
|
||||
DESTINATION_MACHINE = 'shennan@physical-ubuntu-server'
|
||||
ORG = 'mujiannan'
|
||||
PROJECT = 'finance-dbt'
|
||||
DOCKER_REGISTRY = 'docker-registry.mujiannan.com:5001'
|
||||
SMS_URL = 'https://nas.mujiannan.com:5001/webapi/entry.cgi?api=SYNO.Chat.External&method=incoming&version=2&token=%22XO8yFQyi66SStEPY7AALimGbNjMsZE85i3m0UlF0siIQtyn1deqSomp0CUheOwlE%22' // NOSONAR
|
||||
}
|
||||
stages {
|
||||
stage('Pre Check') {
|
||||
steps {
|
||||
script {
|
||||
// Retrieve the latest commit message
|
||||
def String commitMessage = sh(returnStdout: 'true', script: 'git log -1 --pretty=%B').trim()
|
||||
echo "Commit message: ${commitMessage}"
|
||||
// Set a flag based on the commit message
|
||||
if (commitMessage.contains('[SKIP CI]')) {
|
||||
env.SKIP_CI = 'true'
|
||||
}
|
||||
// Set a flag based on the commit message
|
||||
if (
|
||||
commitMessage.contains('[SKIP CD]') || env.SKIP_CI == 'true'
|
||||
) {
|
||||
env.SKIP_CD = 'true'
|
||||
}
|
||||
}
|
||||
script {
|
||||
// Set the image tag, GIT_TAG is preferred, if not available, use the commit hash abbreviated
|
||||
env.IMAGE_TAG = sh(returnStdout: 'true', script: 'git describe --tags --always').trim()
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Build') {
|
||||
when {
|
||||
not {
|
||||
expression {
|
||||
return env.SKIP_CI == 'true'
|
||||
}
|
||||
}
|
||||
}
|
||||
steps {
|
||||
container('kaniko') {
|
||||
// Build the Docker image
|
||||
sh '. ./CI.sh'
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Deploy') {
|
||||
when {
|
||||
not {
|
||||
expression {
|
||||
return env.SKIP_CD == 'true'
|
||||
}
|
||||
}
|
||||
}
|
||||
steps {
|
||||
withKubeConfig([namespace: "${BRANCH_NAME}"]) {
|
||||
container('kubectl') {
|
||||
sh '. ./CD.sh'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
33
README.md
Normal file
33
README.md
Normal file
@ -0,0 +1,33 @@
|
||||
# Welcome to finance dbt project!
|
||||
|
||||
### Prepare
|
||||
```bash
|
||||
cp env.sh env.dev.sh
|
||||
```
|
||||
Change configurations in .env.dev.sh
|
||||
```bash
|
||||
source env.dev.sh
|
||||
```
|
||||
|
||||
### Run dbt
|
||||
Try running the following commands:
|
||||
- dbt deps
|
||||
- dbt run
|
||||
- dbt test
|
||||
|
||||
### Run in docker
|
||||
1. Run pipeline
|
||||
```bash
|
||||
docker run <THIS_IMAGE> <DBT-COMMAND>
|
||||
```
|
||||
2. As document-server
|
||||
```bash
|
||||
docker run -p <PORT>:80 <THIS_IMAGE> sleep 99d
|
||||
```
|
||||
|
||||
### Resources:
|
||||
- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction)
|
||||
- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers
|
||||
- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support
|
||||
- Find [dbt events](https://events.getdbt.com) near you
|
||||
- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices
|
37
dbt_project.yml
Normal file
37
dbt_project.yml
Normal file
@ -0,0 +1,37 @@
|
||||
|
||||
# Name your project! Project names should contain only lowercase characters
|
||||
# and underscores. A good package name should reflect your organization's
|
||||
# name or the intended use of these models
|
||||
name: 'finance_dbt'
|
||||
version: '1.0.0'
|
||||
config-version: 2
|
||||
|
||||
# This setting configures which "profile" dbt uses for this project.
|
||||
profile: 'finance_dbt'
|
||||
|
||||
# These configurations specify where dbt should look for different types of files.
|
||||
# The `model-paths` config, for example, states that models in this project can be
|
||||
# found in the "models/" directory. You probably won't need to change these!
|
||||
model-paths: ["models"]
|
||||
analysis-paths: ["analyses"]
|
||||
test-paths: ["tests"]
|
||||
seed-paths: ["seeds"]
|
||||
macro-paths: ["macros"]
|
||||
snapshot-paths: ["snapshots"]
|
||||
|
||||
clean-targets: # directories to be removed by `dbt clean`
|
||||
- "target"
|
||||
- "dbt_packages"
|
||||
|
||||
|
||||
# Configuring models
|
||||
# Full documentation: https://docs.getdbt.com/docs/configuring-models
|
||||
|
||||
# In this example config, we tell dbt to build all models in the example/
|
||||
# directory as views. These settings can be overridden in the individual model
|
||||
# files using the `{{ config(...) }}` macro.
|
||||
models:
|
||||
finance_dbt:
|
||||
# Config indicated by + and applies to all files under models/example/
|
||||
dw:
|
||||
+materialized: table
|
2
doc-server.sh
Normal file
2
doc-server.sh
Normal file
@ -0,0 +1,2 @@
|
||||
dbt docs generate
|
||||
dbt docs serve
|
3
macros/optimize_table_final.sql
Normal file
3
macros/optimize_table_final.sql
Normal file
@ -0,0 +1,3 @@
|
||||
{% macro optimize_table_final(table_name) %}
|
||||
Optimize Table {{ table_name }} Final
|
||||
{% endmacro %}
|
30
models/dw/dw.dim_date.sql
Normal file
30
models/dw/dw.dim_date.sql
Normal file
@ -0,0 +1,30 @@
|
||||
|
||||
/*
|
||||
日期
|
||||
*/
|
||||
{{
|
||||
config(
|
||||
materialized='table',
|
||||
engine='MergeTree',
|
||||
order_by='date_id'
|
||||
)
|
||||
}}
|
||||
SELECT
|
||||
toUInt32(year(date) * 10000 + month(date) * 100 + day(date)) AS date_id,
|
||||
date AS full_date,
|
||||
toYear(date) AS year,
|
||||
toUInt8(quarter(date)) AS quarter,
|
||||
toUInt8(month(date)) AS month,
|
||||
toUInt8(toDayOfMonth(date)) AS day,
|
||||
toUInt8(week(date)) AS week_num_of_year,
|
||||
toUInt8(toDayOfWeek(date)) AS day_num_of_week
|
||||
FROM (
|
||||
SELECT
|
||||
arrayJoin(
|
||||
arrayMap(
|
||||
x -> toDate('1990-01-01') + x,
|
||||
range(toUInt32(dateDiff('day', toDate('1990-01-01'), toDate('2031-01-01'))))
|
||||
)
|
||||
) AS date
|
||||
)
|
||||
ORDER BY date_id
|
20
models/dw/dw.dim_exchange.sql
Normal file
20
models/dw/dw.dim_exchange.sql
Normal file
@ -0,0 +1,20 @@
|
||||
|
||||
/*
|
||||
交易所
|
||||
*/
|
||||
|
||||
{{ config(materialized='view') }}
|
||||
|
||||
with source_data as (
|
||||
|
||||
select 0 As exchange_id
|
||||
, 'SSE' As ts_exchange_code
|
||||
, '上海证券交易所' As exchange_name
|
||||
Union All
|
||||
select 1 As exchange_id
|
||||
, 'SZSE' As ts_exchange_code
|
||||
, '深圳证券交易所' As exchange_name
|
||||
)
|
||||
|
||||
select *
|
||||
from source_data
|
17
models/dw/dw.dim_hs_calendar.sql
Normal file
17
models/dw/dw.dim_hs_calendar.sql
Normal file
@ -0,0 +1,17 @@
|
||||
/*
|
||||
沪深交易日历
|
||||
*/
|
||||
{{
|
||||
config(
|
||||
materialized='table',
|
||||
engine="MergeTree",
|
||||
order_by='(exchange_id, date_id)',
|
||||
)
|
||||
}}
|
||||
|
||||
Select `Exchange`.`exchange_id` As `exchange_id`
|
||||
, Cast(`StgCalendar`.`cal_date` As Int32) As `date_id`
|
||||
, Cast(`StgCalendar`.`is_open` As Int8) As `is_open`
|
||||
From {{ source('finance', 'stg.tushare_trade_calendar') }} As `StgCalendar` Final
|
||||
All Inner Join {{ ref('dw.dim_exchange') }} As `Exchange` Final
|
||||
On `StgCalendar`.`exchange` = `Exchange`.`ts_exchange_code`
|
45
models/dw/dw.dim_stock.sql
Normal file
45
models/dw/dw.dim_stock.sql
Normal file
@ -0,0 +1,45 @@
|
||||
|
||||
/*
|
||||
股票列表
|
||||
*/
|
||||
{{
|
||||
config(
|
||||
materialized='table',
|
||||
engine='MergeTree',
|
||||
order_by='ts_code'
|
||||
)
|
||||
}}
|
||||
SELECT
|
||||
StgStock.ts_code As ts_code
|
||||
, DimExchange.exchange_id As exchange_id
|
||||
, arrayElement(splitByChar('.', StgStock.`ts_code`),1) As code
|
||||
, StgStock.name As name
|
||||
, StgStock.fullname As full_name
|
||||
, StgStock.enname As name_en
|
||||
, StgStock.cnspell As cn_spell
|
||||
, StgStock.area As area
|
||||
, StgStock.industry As industry
|
||||
, StgStock.market As market
|
||||
, StgStock.curr_type As currency
|
||||
, ListDate.full_date As list_date
|
||||
, Case When StgStock.delist_date Is Null Or StgStock.delist_date = '' Then Null Else DelistDate.full_date End As delist_date
|
||||
, Case
|
||||
When StgStock.list_status = 'L'
|
||||
Then 1
|
||||
Else 0
|
||||
End As is_active
|
||||
, Case
|
||||
When is_hs In ('H', 'S')
|
||||
Then 1
|
||||
Else 0
|
||||
End As is_hongkong_connect
|
||||
, act_name As actual_controller
|
||||
, act_ent_type As actual_controller_enterprise_type
|
||||
, now() As dt
|
||||
From {{ source('finance', 'stg.tushare_stock_basic') }} As StgStock Final
|
||||
Left Join {{ ref("dw.dim_exchange") }} As DimExchange
|
||||
On StgStock.exchange = DimExchange.ts_exchange_code
|
||||
Left Join {{ ref("dw.dim_date") }} As ListDate
|
||||
On StgStock.list_date = Cast(ListDate.date_id As String)
|
||||
Left Join {{ ref("dw.dim_date") }} As DelistDate
|
||||
On StgStock.delist_date = Cast(DelistDate.date_id As String)
|
24
models/dw/dw.dim_time.sql
Normal file
24
models/dw/dw.dim_time.sql
Normal file
@ -0,0 +1,24 @@
|
||||
{{
|
||||
config(
|
||||
materialized='table',
|
||||
engine='MergeTree',
|
||||
order_by='time_id'
|
||||
)
|
||||
}}
|
||||
Select cast(formatDateTime(`full_time`, '%Y%m%d%H%i') As Int64) As time_id
|
||||
, `full_time`
|
||||
, `date_id`
|
||||
, hour(`full_time`) As `hour`
|
||||
, minute(`full_time`) As `minute`
|
||||
From(
|
||||
SELECT
|
||||
arrayJoin(
|
||||
arrayMap(
|
||||
x -> toStartOfMinute(toDateTime(toDate(`full_date`)) + x * 60),
|
||||
range(0, 24 * 60)
|
||||
)
|
||||
) AS `full_time`
|
||||
, `date_id`
|
||||
From {{ ref("dw.dim_date") }}
|
||||
Where `date_id` >= 20090101
|
||||
) As T
|
49
models/dw/dw.fact_stock_daily.sql
Normal file
49
models/dw/dw.fact_stock_daily.sql
Normal file
@ -0,0 +1,49 @@
|
||||
{{
|
||||
config(
|
||||
materialized='incremental',
|
||||
engine='MergeTree',
|
||||
order_by='ts_code, date_id',
|
||||
unique_key=['ts_code', 'date_id'],
|
||||
incremental_strategy='delete+insert',
|
||||
)
|
||||
}}
|
||||
Select
|
||||
StgDaily.ts_code As ts_code,
|
||||
DimDate.date_id As date_id,
|
||||
StgDaily.open As open,
|
||||
StgDaily.high As high,
|
||||
StgDaily.low As low,
|
||||
StgDaily.close As close,
|
||||
StgDaily.pre_close As pre_close,
|
||||
StgDaily.change As change,
|
||||
StgDaily.pct_chg As pct_chg,
|
||||
StgDaily.vol As vol,
|
||||
StgDaily.amount As amount,
|
||||
StgAdjFactor.adj_factor As adj_factor,
|
||||
StgDailyBasic.turnover_rate As turnover_rate,
|
||||
StgDailyBasic.turnover_rate_f As turnover_rate_f,
|
||||
StgDailyBasic.volume_ratio As volume_ratio,
|
||||
StgDailyBasic.pe As pe,
|
||||
StgDailyBasic.pe_ttm As pe_ttm,
|
||||
StgDailyBasic.pb As pb,
|
||||
StgDailyBasic.ps As ps,
|
||||
StgDailyBasic.ps_ttm As ps_ttm,
|
||||
StgDailyBasic.dv_ratio As dv_ratio,
|
||||
StgDailyBasic.dv_ttm As dv_ttm,
|
||||
StgDailyBasic.total_share As total_share,
|
||||
StgDailyBasic.float_share As float_share,
|
||||
StgDailyBasic.free_share As free_share,
|
||||
StgDailyBasic.total_mv As total_mv,
|
||||
StgDailyBasic.circ_mv As circ_mv
|
||||
From {{ source('finance', 'stg.tushare_daily') }} As StgDaily Final
|
||||
Inner Join {{ ref('dw.dim_date') }} As DimDate
|
||||
On StgDaily.trade_date = Cast(DimDate.date_id As String)
|
||||
Any Left Join {{ source('finance', 'stg.tushare_daily_basic') }} As StgDailyBasic
|
||||
On StgDaily.ts_code = StgDailyBasic.ts_code
|
||||
And StgDaily.trade_date = StgDailyBasic.trade_date
|
||||
Any Left Join {{ source('finance', 'stg.tushare_adj_factor') }} As StgAdjFactor
|
||||
On StgDaily.ts_code = StgAdjFactor.ts_code
|
||||
And StgDaily.trade_date = StgAdjFactor.trade_date
|
||||
{% if is_incremental() %}
|
||||
Where StgDaily.trade_date >= (Select cast(max(date_id) As String) From {{ this }})
|
||||
{% endif %}
|
34
models/dw/dw.fact_stock_minute.sql
Normal file
34
models/dw/dw.fact_stock_minute.sql
Normal file
@ -0,0 +1,34 @@
|
||||
{{
|
||||
config(
|
||||
materialized='materialized_view',
|
||||
engine='MergeTree',
|
||||
order_by='ts_code, time_id',
|
||||
unique_key=['ts_code', 'time_id'],
|
||||
partition_by=['cast(floor(time_id / 1000000) As Int32)'],
|
||||
query_settings={"join_algorithm": "'partial_merge'"}
|
||||
)
|
||||
}}
|
||||
|
||||
Select
|
||||
StgMinutes.ts_code As ts_code,
|
||||
DimTime.time_id As time_id,
|
||||
StgMinutes.close As close,
|
||||
StgMinutes.open As open,
|
||||
StgMinutes.high As high,
|
||||
StgMinutes.low As low,
|
||||
StgMinutes.vol As vol,
|
||||
StgMinutes.amount As amount,
|
||||
StgAdjFactor.adj_factor As adj_factor,
|
||||
now() As dt
|
||||
From {{ source('finance', 'stg.tushare_minutes') }} As StgMinutes
|
||||
Inner Join {{ ref('dw.dim_time') }} As DimTime
|
||||
On StgMinutes.trade_time = formatDateTimeInJodaSyntax(full_time, 'yyyy-MM-dd HH:mm:ss')
|
||||
Any Left Join {{ source('finance', 'stg.tushare_adj_factor') }} As StgAdjFactor
|
||||
On StgMinutes.ts_code = StgAdjFactor.ts_code
|
||||
And StgAdjFactor.trade_date = Cast(DimTime.date_id As String)
|
||||
where 1 = 1
|
||||
{% if 'dev' in target.name %}
|
||||
And StgMinutes.trade_time >= formatDateTimeInJodaSyntax(today(), 'yyyy-01-01 00:00:00')
|
||||
{% elif target.name == 'test' %}
|
||||
And StgMinutes.trade_time >= formatDateTimeInJodaSyntax(dateAdd(Year,-1,today()), 'yyyy-01-01 00:00:00')
|
||||
{% endif %}
|
292
models/dw/schema.yml
Normal file
292
models/dw/schema.yml
Normal file
@ -0,0 +1,292 @@
|
||||
|
||||
version: 2
|
||||
|
||||
models:
|
||||
- name: dw.dim_exchange
|
||||
description: "交易所"
|
||||
columns:
|
||||
- name: exchange_id
|
||||
description: "The primary key for this table"
|
||||
tests:
|
||||
- unique
|
||||
- not_null
|
||||
- name: ts_exchange_code
|
||||
description: "交易所代码"
|
||||
tests:
|
||||
- not_null
|
||||
- unique
|
||||
- name: exchange_name
|
||||
description: "交易所名称"
|
||||
tests:
|
||||
- not_null
|
||||
- unique
|
||||
- name: dw.dim_date
|
||||
description: "日期"
|
||||
tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- year
|
||||
- month
|
||||
- day
|
||||
columns:
|
||||
- name: date_id
|
||||
description: "日期键"
|
||||
tests:
|
||||
- unique
|
||||
- not_null
|
||||
- name: full_date
|
||||
description: "日期"
|
||||
tests:
|
||||
- not_null
|
||||
- unique
|
||||
- name: year
|
||||
description: "年"
|
||||
tests:
|
||||
- not_null
|
||||
- name: quarter
|
||||
description: "季度"
|
||||
tests:
|
||||
- not_null
|
||||
- name: month
|
||||
description: "月"
|
||||
tests:
|
||||
- not_null
|
||||
- name: day
|
||||
description: "日"
|
||||
tests:
|
||||
- not_null
|
||||
- name: week_num_of_year
|
||||
description: "年中第几周"
|
||||
tests:
|
||||
- not_null
|
||||
- name: day_num_of_week
|
||||
description: "星期"
|
||||
tests:
|
||||
- not_null
|
||||
- name: dw.dim_time
|
||||
description: "时间"
|
||||
tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- date_id
|
||||
- hour
|
||||
- minute
|
||||
columns:
|
||||
- name: time_id
|
||||
description: "时间键"
|
||||
tests:
|
||||
- unique
|
||||
- not_null
|
||||
- name: full_time
|
||||
description: "时间"
|
||||
tests:
|
||||
- not_null
|
||||
- unique
|
||||
- name: date_id
|
||||
description: "日期键"
|
||||
tests:
|
||||
- not_null
|
||||
- name: hour
|
||||
description: "小时"
|
||||
tests:
|
||||
- not_null
|
||||
- name: minute
|
||||
description: "分钟"
|
||||
tests:
|
||||
- not_null
|
||||
- name: dw.dim_hs_calendar
|
||||
description: "沪深交易日历"
|
||||
tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- exchange_id
|
||||
- date_id
|
||||
columns:
|
||||
- name: exchange_id
|
||||
description: "The primary key for this table"
|
||||
tests:
|
||||
- not_null
|
||||
- name: date_id
|
||||
description: "交易日期键"
|
||||
tests:
|
||||
- not_null
|
||||
- name: is_open
|
||||
description: "是否交易"
|
||||
tests:
|
||||
- not_null
|
||||
- name: dw.dim_stock
|
||||
description: "股票"
|
||||
tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- exchange_id
|
||||
- code
|
||||
columns:
|
||||
- name: ts_code
|
||||
description: "tushare 股票编码"
|
||||
tests:
|
||||
- unique
|
||||
- not_null
|
||||
- name: exchange_id
|
||||
description: "交易所键"
|
||||
tests:
|
||||
- not_null
|
||||
- name: code
|
||||
description: "股票代码"
|
||||
tests:
|
||||
- not_null
|
||||
- name: name
|
||||
description: "股票名称"
|
||||
tests:
|
||||
- not_null
|
||||
- unique
|
||||
- name: full_name
|
||||
description: "股票全称"
|
||||
tests:
|
||||
- not_null
|
||||
- unique
|
||||
- name: name_en
|
||||
description: "股票英文名称"
|
||||
tests:
|
||||
- not_null
|
||||
- name: cn_spell
|
||||
description: "拼音缩写"
|
||||
tests:
|
||||
- not_null
|
||||
- name: area
|
||||
description: "地区"
|
||||
- name: industry
|
||||
description: "行业"
|
||||
- name: market
|
||||
description: "市场类型 (主板/中小板/创业板)"
|
||||
- name: currency
|
||||
description: "交易货币"
|
||||
- name: list_date
|
||||
description: "上市日期"
|
||||
- name: delist_date
|
||||
description: "退市日期"
|
||||
- name: is_active
|
||||
description: "当前是否在市"
|
||||
tests:
|
||||
- not_null
|
||||
- name: is_hongkong_connect
|
||||
description: "是否沪深通标的"
|
||||
tests:
|
||||
- not_null
|
||||
- name: actual_controller
|
||||
description: "实际控制人"
|
||||
- name: actual_controller_enterprise_type
|
||||
description: "实际控制人企业类型"
|
||||
- name: dt
|
||||
description: "更新时间"
|
||||
- name: dw.fact_stock_daily
|
||||
description: "股票日数据"
|
||||
tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- ts_code
|
||||
- date_id
|
||||
columns:
|
||||
- name: ts_code
|
||||
description: "股票键"
|
||||
tests:
|
||||
- not_null
|
||||
- name: date_id
|
||||
description: "日期键"
|
||||
tests:
|
||||
- not_null
|
||||
- name: open
|
||||
description: "开盘价"
|
||||
- name: high
|
||||
description: "最高价"
|
||||
- name: low
|
||||
description: "最低价"
|
||||
- name: close
|
||||
description: "收盘价"
|
||||
- name: pre_close
|
||||
description: "昨收价"
|
||||
- name: change
|
||||
description: "涨跌额"
|
||||
- name: pct_chg
|
||||
description: "涨跌幅"
|
||||
- name: vol
|
||||
description: "成交量"
|
||||
- name: amount
|
||||
description: "成交额"
|
||||
- name: adj_factor
|
||||
description: "复权因子"
|
||||
- name: turnover_rate
|
||||
description: "换手率"
|
||||
- name: turnover_rate_free
|
||||
description: "换手率(自由流通股)"
|
||||
- name: volume_ratio
|
||||
description: "量比"
|
||||
- name: pe
|
||||
description: "市盈率(总市值/净利润, 亏损的PE为空)"
|
||||
- name: pe_ttm
|
||||
description: "市盈率(TTM)"
|
||||
- name: pb
|
||||
description: "市净率(总市值/净资产)"
|
||||
- name: ps
|
||||
description: "市销率"
|
||||
- name: ps_ttm
|
||||
description: "市销率(TTM)"
|
||||
- name: dv_ratio
|
||||
description: "股息率"
|
||||
- name: dv_ttm
|
||||
description: "股息率(TTM)"
|
||||
- name: total_share
|
||||
description: "总股本(万股)"
|
||||
- name: float_share
|
||||
description: "流通股本(万股)"
|
||||
- name: free_share
|
||||
description: "自由流通股本(万)"
|
||||
- name: total_mv
|
||||
description: "总市值(万元)"
|
||||
- name: circ_mv
|
||||
description: "流通市值(万元)"
|
||||
- name: dw.fact_stock_minute
|
||||
description: "股票分钟数据"
|
||||
tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- ts_code
|
||||
- time_id
|
||||
columns:
|
||||
- name: ts_code
|
||||
description: "股票键"
|
||||
data_type: "String"
|
||||
tests:
|
||||
- not_null
|
||||
- name: time_id
|
||||
description: "时间键"
|
||||
data_type: "Int64"
|
||||
tests:
|
||||
- not_null
|
||||
- name: open
|
||||
description: "开盘价"
|
||||
data_type: "Nullable(Decimal(16,2))"
|
||||
- name: high
|
||||
description: "最高价"
|
||||
data_type: "Nullable(Decimal(16,2))"
|
||||
- name: low
|
||||
description: "最低价"
|
||||
data_type: "Nullable(Decimal(16,2))"
|
||||
- name: close
|
||||
description: "收盘价"
|
||||
data_type: "Nullable(Decimal(16,2))"
|
||||
- name: vol
|
||||
description: "成交量"
|
||||
data_type: "Nullable(Int32)"
|
||||
- name: amount
|
||||
description: "成交额"
|
||||
data_type: "Nullable(Decimal(18,2))"
|
||||
- name: adj_factor
|
||||
description: "复权因子"
|
||||
data_type: "Decimal(16,6)"
|
||||
tests:
|
||||
- not_null
|
||||
- name: dt
|
||||
description: "时间"
|
||||
data_type: "DateTime"
|
||||
|
253
models/finance_source.yml
Normal file
253
models/finance_source.yml
Normal file
@ -0,0 +1,253 @@
|
||||
version: 2
|
||||
sources:
|
||||
- name: finance
|
||||
schema: '{{ env_var("FINANCE_CLICKHOUSE_DATABASE") }}'
|
||||
description: "The source for financial data"
|
||||
tables:
|
||||
- name: stg.tushare_trade_calendar
|
||||
description: >
|
||||
The source table for the trade calendar data from Tushare
|
||||
tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- exchange
|
||||
- cal_date
|
||||
columns:
|
||||
- name: exchange
|
||||
description: "The exchange code"
|
||||
tests:
|
||||
- not_null
|
||||
- name: cal_date
|
||||
description: "The date"
|
||||
tests:
|
||||
- not_null
|
||||
- name: is_open
|
||||
description: "Whether the exchange is open"
|
||||
tests:
|
||||
- not_null
|
||||
- name: pretrade_date
|
||||
description: "The previous trading date"
|
||||
- name: stg.tushare_stock_basic
|
||||
columns:
|
||||
- name: ts_code
|
||||
description: TS代码
|
||||
tests:
|
||||
- not_null
|
||||
- name: symbol
|
||||
description: 股票代码
|
||||
- name: name
|
||||
description: 股票名称
|
||||
- name: area
|
||||
description: 所在地域
|
||||
- name: industry
|
||||
description: 所属行业
|
||||
- name: fullname
|
||||
description: 股票全称
|
||||
- name: enname
|
||||
description: 英文全称
|
||||
- name: cnspell
|
||||
description: 拼音缩写
|
||||
- name: market
|
||||
description: 市场类型 (主板/中小板/创业板)
|
||||
- name: exchange
|
||||
description: 交易所代码
|
||||
tests:
|
||||
- not_null
|
||||
- name: curr_type
|
||||
description: 交易货币
|
||||
- name: list_status
|
||||
description: 上市状态: L上市 D退市 P暂停上市
|
||||
- name: list_date
|
||||
description: 上市日期
|
||||
- name: delist_date
|
||||
description: 退市日期
|
||||
- name: is_hs
|
||||
description: 是否沪深港通标的,N否 H沪股通 S深股通
|
||||
- name: act_name
|
||||
description: 实控人名称
|
||||
- name: act_ent_type
|
||||
description: 实控人企业性质
|
||||
- name: dt
|
||||
description: 最后修改时间
|
||||
unique_composite:
|
||||
- name: unique_exchange_ts_code
|
||||
columns:
|
||||
- exchange
|
||||
- ts_code
|
||||
- name: stg.tushare_daily
|
||||
description: "沪深股票交易日线数据"
|
||||
tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- ts_code
|
||||
- trade_date
|
||||
columns:
|
||||
- name: ts_code
|
||||
description: TS代码
|
||||
tests:
|
||||
- not_null
|
||||
- name: trade_date
|
||||
description: 交易日期
|
||||
tests:
|
||||
- not_null
|
||||
- name: open
|
||||
description: 开盘价
|
||||
tests:
|
||||
- not_null
|
||||
- name: high
|
||||
description: 最高价
|
||||
tests:
|
||||
- not_null
|
||||
- name: low
|
||||
description: 最低价
|
||||
tests:
|
||||
- not_null
|
||||
- name: close
|
||||
description: 收盘价
|
||||
tests:
|
||||
- not_null
|
||||
- name: pre_close
|
||||
description: 昨收价
|
||||
tests:
|
||||
- not_null
|
||||
- name: change
|
||||
description: 涨跌额
|
||||
tests:
|
||||
- not_null
|
||||
- name: pct_chg
|
||||
description: 涨跌幅
|
||||
tests:
|
||||
- not_null
|
||||
- name: vol
|
||||
description: 成交量
|
||||
tests:
|
||||
- not_null
|
||||
- name: amount
|
||||
description: 成交额
|
||||
- name: stg.tushare_daily_basic
|
||||
description: "沪深股票每日指标数据"
|
||||
tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- ts_code
|
||||
- trade_date
|
||||
|
||||
columns:
|
||||
- name: ts_code
|
||||
description: "股票代码"
|
||||
data_type: String
|
||||
- name: trade_date
|
||||
description: "交易日期"
|
||||
data_type: String
|
||||
- name: close
|
||||
description: "当日收盘价"
|
||||
data_type: Nullable(Float32)
|
||||
- name: turnover_rate
|
||||
description: "换手率(%)"
|
||||
data_type: Nullable(Float32)
|
||||
- name: turnover_rate_f
|
||||
description: "换手率(自由流通股)"
|
||||
data_type: Nullable(Float32)
|
||||
- name: volume_ratio
|
||||
description: "量比"
|
||||
data_type: Nullable(Float32)
|
||||
- name: pe
|
||||
description: "市盈率(总市值/净利润, 亏损的PE为空)"
|
||||
data_type: Nullable(Float32)
|
||||
- name: pe_ttm
|
||||
description: "市盈率(TTM,亏损的PE为空)"
|
||||
data_type: Nullable(Float32)
|
||||
- name: pb
|
||||
description: "市净率(总市值/净资产)"
|
||||
data_type: Nullable(Float32)
|
||||
- name: ps
|
||||
description: "市销率"
|
||||
data_type: Nullable(Float32)
|
||||
- name: ps_ttm
|
||||
description: "市销率(TTM)"
|
||||
data_type: Nullable(Float32)
|
||||
- name: dv_ratio
|
||||
description: "股息率 (%)"
|
||||
data_type: Nullable(Float32)
|
||||
- name: dv_ttm
|
||||
description: "股息率(TTM)(%)"
|
||||
data_type: Nullable(Float32)
|
||||
- name: total_share
|
||||
description: "总股本 (万股)"
|
||||
data_type: Nullable(Float32)
|
||||
- name: float_share
|
||||
description: "流通股本 (万股)"
|
||||
data_type: Nullable(Float32)
|
||||
- name: free_share
|
||||
description: "自由流通股本 (万)"
|
||||
data_type: Nullable(Float32)
|
||||
- name: total_mv
|
||||
description: "总市值 (万元)"
|
||||
data_type: Nullable(Float32)
|
||||
- name: circ_mv
|
||||
description: "流通市值(万元)"
|
||||
data_type: Nullable(Float32)
|
||||
- name: stg.tushare_adj_factor
|
||||
description: "沪深股票复权因子"
|
||||
tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- ts_code
|
||||
- trade_date
|
||||
columns:
|
||||
- name: ts_code
|
||||
description: "股票代码"
|
||||
data_type: String
|
||||
- name: trade_date
|
||||
description: "交易日期"
|
||||
data_type: String
|
||||
- name: adj_factor
|
||||
description: "复权因子"
|
||||
data_type: Nullable(Decimal(16, 6))
|
||||
- name: dt
|
||||
description: "更新日期"
|
||||
data_type: DateTime
|
||||
- name: stg.tushare_minutes
|
||||
description: "沪深分钟级交易数据"
|
||||
tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- ts_code
|
||||
- trade_time
|
||||
config:
|
||||
where: "left(trade_time,7)>=concat(cast(year(today())-1 As String), '-01')"
|
||||
columns:
|
||||
- name: ts_code
|
||||
description: "The stock code."
|
||||
data_type: String
|
||||
tests:
|
||||
- not_null
|
||||
- name: trade_time
|
||||
description: "The trading time."
|
||||
data_type: String
|
||||
tests:
|
||||
- not_null
|
||||
- name: close
|
||||
description: "The closing price."
|
||||
data_type: Nullable(Float32)
|
||||
- name: open
|
||||
description: "The opening price."
|
||||
data_type: Nullable(Float32)
|
||||
- name: high
|
||||
description: "The highest price."
|
||||
data_type: Nullable(Float32)
|
||||
- name: low
|
||||
description: "The lowest price."
|
||||
data_type: Nullable(Float32)
|
||||
- name: vol
|
||||
description: "The volume of trades."
|
||||
data_type: Nullable(Float32)
|
||||
- name: amount
|
||||
description: "The amount of trades."
|
||||
data_type: Nullable(Float32)
|
||||
meta:
|
||||
engine: ReplacingMergeTree
|
||||
partition_by: "left(trade_time, 7)"
|
||||
order_by: "(ts_code, trade_time)"
|
||||
settings:
|
||||
index_granularity: 8192
|
4
package-lock.yml
Normal file
4
package-lock.yml
Normal file
@ -0,0 +1,4 @@
|
||||
packages:
|
||||
- package: dbt-labs/dbt_utils
|
||||
version: 1.1.1
|
||||
sha1_hash: a158c48c59c2bb7d729d2a4e215aabe5bb4f3353
|
3
packages.yml
Normal file
3
packages.yml
Normal file
@ -0,0 +1,3 @@
|
||||
packages:
|
||||
- package: dbt-labs/dbt_utils
|
||||
version: 1.1.1
|
36
profiles.yml
Normal file
36
profiles.yml
Normal file
@ -0,0 +1,36 @@
|
||||
finance_dbt:
|
||||
target: "{{ env_var('FINANCE_DBT_TARGET') }}"
|
||||
outputs:
|
||||
.template: &default_template # 定义锚点
|
||||
type: clickhouse
|
||||
schema: "{{ env_var('FINANCE_CLICKHOUSE_DATABASE') }}"
|
||||
# optional
|
||||
driver: http
|
||||
host: "{{ env_var('FINANCE_CLICKHOUSE_HOST') }}"
|
||||
port: "{{ env_var('FINANCE_CLICKHOUSE_HTTP_PORT') | as_number }}"
|
||||
user: "{{ env_var('FINANCE_CLICKHOUSE_USER') }}"
|
||||
password: "{{ env_var('FINANCE_CLICKHOUSE_PASSWORD') }}"
|
||||
verify: True
|
||||
secure: False
|
||||
retries: 1
|
||||
compression: gzip
|
||||
connect_timeout: 10
|
||||
send_receive_timeout: 300
|
||||
cluster_mode: False
|
||||
use_lw_deletes: True
|
||||
check_exchange: True
|
||||
local_suffix: _local
|
||||
allow_automatic_deduplication: True
|
||||
custom_settings: {}
|
||||
|
||||
# Native (clickhouse-driver) connection settings
|
||||
sync_request_timeout: 5
|
||||
compress_block_size: 1048576
|
||||
dev:
|
||||
<<: *default_template # 引用锚点
|
||||
test:
|
||||
<<: *default_template # 引用锚点
|
||||
prod:
|
||||
<<: *default_template # 引用锚点
|
||||
local_dev:
|
||||
<<: *default_template # 引用锚点
|
2
requirements.txt
Normal file
2
requirements.txt
Normal file
@ -0,0 +1,2 @@
|
||||
dbt==1.0.0.37.0
|
||||
dbt-clickhouse==1.7.3
|
Loading…
Reference in New Issue
Block a user