finance-dbt/models/ods/ods.tushare_minutes.sql

{{ 
    config(
        materialized='incremental',
        engine="MergeTree",
        order_by='(time_id, ts_code)',
        unique_key=['time_id', 'ts_code'],
        partition_by=['toYYYYMM(toDateTime(time_id))'],
        incremental_strategy='delete+insert',
        query_settings={
            "join_algorithm": "'full_sorting_merge'", 
            "max_bytes_before_external_sort": "'1000M'",
            "max_bytes_before_external_group_by":"'1000M'",
        }
    )
}}

Select 
    StgMinutes.ts_code As ts_code,
    toYYYYMMDDhhmmss(toDateTime(`StgMinutes`.`trade_time`)) As time_id,
    StgMinutes.close As close,
    StgMinutes.open As open,
    StgMinutes.high As high,
    StgMinutes.low As low,
    StgMinutes.vol As vol,
    StgMinutes.amount As amount
From {{ source('finance', 'stg.tushare_minutes') }} As StgMinutes 
where 1 = 1
{% if is_incremental() %}
    And StgMinutes.trade_time >= (Select formatDateTimeInJodaSyntax(toStartOfDay(YYYYMMDDhhmmssToDateTime(max(time_id))), 'yyyy-MM-dd HH:mm:ss') From {{ this }})
{% endif %}
{% if 'dev' in target.name %}
    And StgMinutes.trade_time >= formatDateTimeInJodaSyntax(dateAdd(Month,-1,today()), 'yyyy-01-01 00:00:00')
{% elif target.name == 'test' %}
    And StgMinutes.trade_time >= formatDateTimeInJodaSyntax(dateAdd(Year,-1,today()), 'yyyy-01-01 00:00:00')
{% endif %}