diff --git a/README.md b/README.md index a741d03..b109120 100644 --- a/README.md +++ b/README.md @@ -17,3 +17,12 @@ TRUNCATE TABLE raw.performance_review_sub_scoring; TRUNCATE TABLE raw.performance_review_total_scoring; TRUNCATE TABLE raw.positions; TRUNCATE TABLE raw.workers; + + +DROP TABLE raw.absenteisme; +DROP TABLE raw.departments; +DROP TABLE raw.performance_review_steps; +DROP TABLE raw.performance_review_sub_scoring; +DROP TABLE raw.performance_review_total_scoring; +DROP TABLE raw.positions; +DROP TABLE raw.workers; diff --git a/transform/models/tap_spreadsheets_anywhere/job_change.sql b/transform/models/tap_spreadsheets_anywhere/job_change.sql index f0d4883..29f1bc1 100644 --- a/transform/models/tap_spreadsheets_anywhere/job_change.sql +++ b/transform/models/tap_spreadsheets_anywhere/job_change.sql @@ -24,7 +24,10 @@ deduplicated_positions AS ( * FROM {{ source('tap_spreadsheets_anywhere', 'positions') }} WHERE _sdc_deleted_at IS NULL - ORDER BY assigned_employee_id, assigned_employee_effective_date, _sdc_received_at DESC + ORDER BY assigned_employee_id, + assigned_employee_effective_date, + CASE WHEN primary_position = 'Yes' THEN 1 ELSE 0 END DESC, + _sdc_received_at DESC ), transformed_worker AS ( diff --git a/transform/models/tap_spreadsheets_anywhere/worker.sql b/transform/models/tap_spreadsheets_anywhere/worker.sql index aad1e19..17f9d35 100644 --- a/transform/models/tap_spreadsheets_anywhere/worker.sql +++ b/transform/models/tap_spreadsheets_anywhere/worker.sql @@ -10,6 +10,8 @@ WITH latest_workers AS ( FROM {{ source('tap_spreadsheets_anywhere', 'workers') }} WHERE _sdc_deleted_at IS NULL AND user_id IS NOT NULL and user_id != '' -- Skipping empty user_ids + -- Skipping users with non-numeric user_id, as they are not valid + AND user_id ~ '^[0-9]+$' ORDER BY user_id, _sdc_received_at DESC ), @@ -20,8 +22,8 @@ latest_positions AS ( SELECT DISTINCT ON (assigned_employee_id) * FROM {{ source('tap_spreadsheets_anywhere', 'positions') }} WHERE _sdc_deleted_at IS NULL - AND primary_position = 'Yes' - ORDER BY assigned_employee_id, _sdc_received_at DESC, assigned_employee_effective_date DESC + -- AND primary_position = 'Yes' -- Removed this filter, as there are employees with only non-primary positions + ORDER BY assigned_employee_id, _sdc_received_at DESC, assigned_employee_effective_date DESC, CASE WHEN primary_position = 'Yes' THEN 1 ELSE 0 END DESC ), -- Get the first position for each worker, to set the hire date