Skip to content

Job request: 18300

Organisation:
The London School of Hygiene & Tropical Medicine
Workspace:
healthcare_utilisation_openprompt
ID:
mbn3ibss6tl76lhn
View repo View project.yaml

This page shows the technical details of what happened when the authorised researcher Liang-Yu Lin requested one or more actions to be run against real patient data in the project, within a secure environment.

By cross-referencing the list of jobs with the pipeline section below, you can infer what security level various outputs were written to. Researchers can never directly view outputs marked as highly_sensitive ; they can only request that code runs against them. Outputs marked as moderately_sensitive can be viewed by an approved researcher by logging into a highly secure environment. Only outputs marked as moderately_sensitive can be requested for release to the public, via a controlled output review service.

Jobs

Pipeline

Show project.yaml 
version: '3.0'

expectations:
  population_size: 5000

actions:

  generate_long_covid_exposure_dataset:
    run: 
      databuilder:v0 generate-dataset
        analysis/dataset_definition_unmatched_exp_lc.py
        --output output/dataset_exp_lc_unmatched.csv
    outputs:
      highly_sensitive:
        cohort: output/dataset_exp_lc_unmatched.csv

  generate_list_gp_use_long_covid_dx:
    run: 
      databuilder:v0 generate-dataset
        analysis/dataset_definition_lc_gp_list.py
        --output output/dataset_lc_gp_list.csv
    outputs:
      highly_sensitive:
        cohort: output/dataset_lc_gp_list.csv

  generate_dataset_comparator_exclude_gp_no_long_covid:
    needs: [generate_list_gp_use_long_covid_dx]
    run: 
      databuilder:v0 generate-dataset
        analysis/dataset_definition_unmatched_comparator.py
        --output output/dataset_comparator_unmatched.csv
    outputs:
      highly_sensitive:
        cohort: output/dataset_comparator_unmatched.csv

  test_matching:
    run:
      python:latest python analysis/match_test.py
    needs: [generate_dataset_comparator_exclude_gp_no_long_covid, generate_long_covid_exposure_dataset]
    outputs: 
      highly_sensitive:
        matched_cases: output/matched_cases_stp.csv
        matched_matches: output/matched_matches_stp.csv
        matched_all: output/matched_combined_stp.csv
      moderately_sensitive: 
        matching_report: output/matching_report_stp.txt

  import_matched_exposure:
    run: >
       databuilder:v0
        generate-dataset analysis/dataset_definition_matched_cases.py
        --output output/matched_cases_with_ehr.csv
    needs: [test_matching]
    outputs: 
      highly_sensitive:
        cohort: output/matched_cases_with_ehr.csv

  import_matched_controls:
    run: >
       databuilder:v0
        generate-dataset analysis/dataset_definition_matched_control.py
        --output output/matched_control_with_ehr.csv
    needs: [test_matching]
    outputs: 
      highly_sensitive:
        cohort: output/matched_control_with_ehr.csv

  generate_historical_exp_data:
    run: 
      databuilder:v0 generate-dataset analysis/dataset_definition_hx_unmatched_exp_lc.py
        --output output/hx_unmatched_exp.csv
    outputs:
      highly_sensitive:
        hx_cohort: output/hx_unmatched_exp.csv
  
  generate_historical_comp_data_exclude_gp_no_long_covid:
    needs: [generate_list_gp_use_long_covid_dx]
    run: 
      databuilder:v0 generate-dataset analysis/dataset_definition_hx_unmatched_com_no_lc.py
        --output output/hx_dataset_comp_unmatched.csv
    outputs:
      highly_sensitive:
        hx_cohort: output/hx_dataset_comp_unmatched.csv

  historical_matching:
    run:
      python:latest python analysis/match_historical.py
    needs: [generate_historical_exp_data, generate_historical_comp_data_exclude_gp_no_long_covid]
    outputs: 
      highly_sensitive:
        matched_cases: output/matched_cases_historical.csv
        matched_matches: output/matched_matches_historical.csv
        matched_all: output/matched_combined_historical.csv
      moderately_sensitive: 
        matching_report: output/matching_report_historical.txt

  import_matched_historical_exposure:
    run: >
       databuilder:v0
        generate-dataset analysis/dataset_definition_hx_matched_exp_lc.py
        --output output/hx_matched_cases_with_ehr.csv
    needs: [historical_matching]
    outputs: 
      highly_sensitive:
        cohort: output/hx_matched_cases_with_ehr.csv

  import_matched_historical_controls:
    run: >
       databuilder:v0
        generate-dataset analysis/dataset_definition_hx_matched_comp.py
        --output output/hx_matched_control_with_ehr.csv
    needs: [historical_matching]
    outputs: 
      highly_sensitive:
        cohort: output/hx_matched_control_with_ehr.csv

# Reporting:

  report01_matched_datasets:
    needs: [import_matched_exposure, import_matched_controls]
    run: 
      r:latest analysis/st01_report_matched.R
    outputs: 
      moderately_sensitive: 
        matched_table: output/st01_matched_numbers_table.csv
        explore_vax_fig: output/st1_exporing_vax_index_date.png
        missing_table: output/missing_distribution_table.csv
        missing_pattern: output/missing_pattern_current.png

  report02_hx_matched_datasets:
    needs: [import_matched_historical_exposure, import_matched_historical_controls]
    run: 
      r:latest analysis/st02_report_matched_historical.R
    outputs: 
      moderately_sensitive: 
        matched_table: output/hx_matched_numbers_table.csv

  report_03_hurdle_model:
    needs: [import_matched_exposure, import_matched_controls]
    run: 
      r:latest analysis/st03_hurdle_model.R
    outputs: 
      moderately_sensitive: 
        hurdle_all: output/st_03_result_monthly_visit_hurdle.csv
        
  report_03_1_hurdle_model_predict:
    needs: [import_matched_exposure, import_matched_controls]
    run: 
      r:latest analysis/st03_hurdle_model_predict.R
    outputs: 
      moderately_sensitive: 
        hurdle_all: output/st_03_result_cumulative_visit_hurdle.csv
        hurdle_gp: output/st_03_gp_result_cumulative_visit_hurdle.csv
        hurdle_hos: output/st_03_hos_result_cumulative_visit_hurdle.csv
        hurdle_ae: output/st_03_ae_result_cumulative_visit_hurdle.csv
        
  report_04_hurdle_model_plot:
    needs: [report_03_1_hurdle_model_predict]
    run: 
      r:latest analysis/st04_plot_hurdle_visit.R
    outputs: 
      moderately_sensitive: 
        crude: output/st_04_crude_healthcare_visit.png
        partial: output/st_04_partial_adj_healthcare_visit.png
        full: output/st_04_full_adj_healthcare_visit.png
  
  report_03_2_hurdle_model_sugroup_predict:
    needs: [import_matched_exposure, import_matched_controls]
    run: 
      r:latest analysis/st03_sub_hurdle_model_predict.R
    outputs: 
      moderately_sensitive: 
        covid_hos_visit: "output/st_03_result_hos_cumulative_visit_hurdle.csv"
        no_covid_hos_visit: "output/st_03_result_no_hos_cumulative_visit_hurdle.csv"  
        
  report_03_3_twopart_model_predict:
    needs: [import_matched_exposure, import_matched_controls]
    run: 
      r:latest analysis/st03_two_part_model.R
    outputs: 
      moderately_sensitive: 
        gp_costs: "output/st_04_result_cumulative_cost_full_2pm.csv"

Timeline

  • Created:

  • Started:

  • Finished:

  • Runtime: 02:39:36

These timestamps are generated and stored using the UTC timezone on the TPP backend.

Job information

Status
Failed
Backend
TPP
Workspace
healthcare_utilisation_openprompt
Requested by
Liang-Yu Lin
Branch
main
Force run dependencies
No
Git commit hash
333f7df
Requested actions
  • import_matched_historical_exposure
  • import_matched_historical_controls

Code comparison

Compare the code used in this Job Request