Minor fixes after workshop

Worker: Also consider non-primary positions for the department
identification
Worker: Don't load users with non-numeric IDs
Job change: Select position with primary = Yes, if it exists

README.md

@@ -17,3 +17,12 @@ TRUNCATE TABLE raw.performance_review_sub_scoring;
 TRUNCATE TABLE raw.performance_review_total_scoring;
 TRUNCATE TABLE raw.positions;
 TRUNCATE TABLE raw.workers;
+
+
+DROP TABLE raw.absenteisme;
+DROP TABLE raw.departments;
+DROP TABLE raw.performance_review_steps;
+DROP TABLE raw.performance_review_sub_scoring;
+DROP TABLE raw.performance_review_total_scoring;
+DROP TABLE raw.positions;
+DROP TABLE raw.workers;

transform/models/tap_spreadsheets_anywhere/job_change.sql

@@ -24,7 +24,10 @@ deduplicated_positions AS (
         *
     FROM {{ source('tap_spreadsheets_anywhere', 'positions') }}
     WHERE _sdc_deleted_at IS NULL
-    ORDER BY assigned_employee_id, assigned_employee_effective_date, _sdc_received_at DESC
+    ORDER BY assigned_employee_id, 
+        assigned_employee_effective_date, 
+        CASE WHEN primary_position = 'Yes' THEN 1 ELSE 0 END DESC,
+        _sdc_received_at DESC
 ),
 
 transformed_worker AS (

transform/models/tap_spreadsheets_anywhere/worker.sql

@@ -10,6 +10,8 @@ WITH latest_workers AS (
     FROM {{ source('tap_spreadsheets_anywhere', 'workers') }}
     WHERE _sdc_deleted_at IS NULL
     AND user_id IS NOT NULL and user_id != '' -- Skipping empty user_ids
+    -- Skipping users with non-numeric user_id, as they are not valid
+    AND user_id ~ '^[0-9]+$'
     ORDER BY user_id, _sdc_received_at DESC
 ),
 
@@ -20,8 +22,8 @@ latest_positions AS (
     SELECT DISTINCT ON (assigned_employee_id) *
     FROM {{ source('tap_spreadsheets_anywhere', 'positions') }}
     WHERE _sdc_deleted_at IS NULL
-	AND primary_position = 'Yes'
-    ORDER BY assigned_employee_id, _sdc_received_at DESC, assigned_employee_effective_date DESC
+	-- AND primary_position = 'Yes' -- Removed this filter, as there are employees with only non-primary positions
+    ORDER BY assigned_employee_id, _sdc_received_at DESC, assigned_employee_effective_date DESC, CASE WHEN primary_position = 'Yes' THEN 1 ELSE 0 END DESC
 ),
 
 -- Get the first position for each worker, to set the hire date