Skip to content

Job request: 5380

Organisation:
University of Manchester
Workspace:
service_eval_work
ID:
e6bmzvllyiyiwbt5
View repo View project.yaml

This page shows the technical details of what happened when the authorised researcher Ya-Ting Yang requested one or more actions to be run against real patient data in the project, within a secure environment.

By cross-referencing the list of jobs with the pipeline section below, you can infer what security level various outputs were written to. Researchers can never directly view outputs marked as highly_sensitive ; they can only request that code runs against them. Outputs marked as moderately_sensitive can be viewed by an approved researcher by logging into a highly secure environment. Only outputs marked as moderately_sensitive can be requested for release to the public, via a controlled output review service.

Jobs

Pipeline

Show project.yaml 
version: '3.0'

expectations:
  population_size: 1000

actions:

  generate_study_population:
    run: cohortextractor:latest generate_cohort --study-definition study_definition --index-date-range "2019-01-01 to today by month" --skip-existing --output-dir=output/measures --output-format=csv.gz
    outputs:
      highly_sensitive:
        cohort: output/measures/input_*.csv.gz
  
  generate_study_population_infection:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_infection --index-date-range "2019-01-01 to today by month" --skip-existing --output-dir=output/measures --output-format=csv.gz
    outputs:
      highly_sensitive:
        cohort: output/measures/input_infection_*.csv.gz

  generate_study_population_elderly:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_elderly 
      --output-format=csv.gz
    outputs:
      highly_sensitive:
        cohort: output/input_elderly.csv.gz

  generate_measures:
    run: cohortextractor:latest generate_measures --study-definition study_definition --skip-existing --output-dir=output/measures
    needs: [generate_study_population]
    outputs:
      moderately_sensitive:
        measure_csv: output/measures/measure_*.csv
  
  generate_measures_infection:
    run: cohortextractor:latest generate_measures --study-definition study_definition_infection --skip-existing --output-dir=output/measures
    needs: [generate_study_population_infection]
    outputs:
      moderately_sensitive:
        measure_csv: output/measures/measure_infection_*.csv
        
  # describe_elderly_agedis:
  #   run: r:latest analysis/tables/gen_csv_age_check.R
  #   needs: [generate_study_population_elderly]
  #   outputs:
  #     moderately_sensitive:
  #       agetable: output/age_quant.csv

  # describe:
  #   run: r:latest analysis/plot/overall_ab_prescribing.R
  #   needs: [generate_measures]
  #   outputs:
  #     moderately_sensitive:
  #       cohort: output/overall.png
  #       boxplot: output/overallbox.png

  describe_percentile:
    run: r:latest analysis/plot/overall_ab_prescribing_2575percentile.R
    needs: [generate_measures]
    outputs:
      moderately_sensitive:
        percentile: output/overall_25th_75th_percentile.png

  # describe_starpu:
  #   run: r:latest analysis/plot/starpu_ab_prescribing.R
  #   needs: [generate_measures]
  #   outputs:
  #     moderately_sensitive:
  #       cohort: output/starpuline.png
  #       boxplot: output/starpubox.png
  
  # generate_notebook_starpu:
  #   run: jupyter:latest jupyter nbconvert /workspace/analysis/starpu.ipynb --execute --to html --output-dir=/workspace/output --ExecutePreprocessor.timeout=86400
  #   needs: [generate_measures]
  #   outputs:
  #     moderately_sensitive:
  #       notebook: output/starpu.html 
  #       figures: output/*
  #       #tables: output/tables/*
  #       #csvs: output/*/* # two possible subfolders
  #       #text: output/text/*
  
  describe_service_eval_baseline_table:
    run: r:latest analysis/tables/baseline_table.R
    needs: [generate_study_population]
    outputs:
      moderately_sensitive:
        baseline_overall: output/overall_counts_blt.csv
        baseline_table: output/blt_one_random_obs_perpat.csv

  # describe_consultation_rate:
  #   run: r:latest analysis/plot/incident_consultation_age_stacked_barchart.R
  #   needs: [generate_measures]
  #   outputs:
  #      moderately_sensitive:
  #       bar1: output/consult_age_UTI.png
  #       bar2: output/consult_age_LRTI.png
  #       bar3: output/consult_age_URTI.png
  #       bar4: output/consult_age_sinusitis.png
  #       bar5: output/consult_age_ot_externa.png
  #       bar6: output/consult_age_otmedia.png
  #       bar7: output/consult_age_repeatedUTI.png
       


  # describe_consultation_prescribed:
  #   run: r:latest analysis/plot/consultation_prescibed_percentage.R
  #   needs: [generate_study_population]
  #   outputs:
  #      moderately_sensitive:
  #       bar1: output/prescribed_percentage_UTI.png
  #       csvs: output/uti_prescrib_check.csv
     

  #generate_notebook_starpu:
  #  run: jupyter:latest jupyter nbconvert /workspace/analysis/starpu.ipynb --execute --to html --output-dir=/workspace/output/hospitalisation_risk --ExecutePreprocessor.timeout=86400
  #  needs: [generate_measures]
  #  outputs:
  #    moderately_sensitive:
  #      notebook: output/hospitalisation_risk/starpu.html 
  #      figures: output/hospitalisation_risk/*
        #tables: output/tables/*
        #csvs: output/*/* # two possible subfolders
        #text: output/text/*

  # generate_notebook_hospitalisation_analysis:
  #   run: jupyter:latest jupyter nbconvert /workspace/analysis/hospitalisation_analysis.ipynb --execute --to html --output-dir=/workspace/output/hospitalisation_risk --ExecutePreprocessor.timeout=86400
  #   needs: [generate_study_population]
  #   outputs:
  #   moderately_sensitive:
  #       notebook: output/hospitalisation_risk/hospitalisation_analysis.html 
  #       figures: output/hospitalisation_risk/*
    
  # generate_notebook_hospitalisation_analysis:
  #   run: jupyter:latest jupyter nbconvert /workspace/analysis/hospitalisation_analysis.ipynb --execute --to html --output-dir=/workspace/output/hospitalisation_risk --ExecutePreprocessor.timeout=86400
  #   needs: [generate_study_population]
  #   outputs:
  #     moderately_sensitive:
  #       notebook: output/hospitalisation_risk/hospitalisation_analysis.html 
  #       figures: output/hospitalisation_risk/*

  describe_prior_ab_12mb4:
    run: r:latest analysis/plot/ab_1yb4_stackedbar_2.R
    needs: [generate_study_population]
    outputs:
       moderately_sensitive:
        plot: output/AB_1yb4_line.jpeg
        plot_sex: output/AB_1yb4_SEX.jpeg
        #count_table: output/prior_ab_by_month.csv      


  describe_consultation_rate_all:
    run: r:latest analysis/plot/incident_consultation_by_age_infection.R
    needs: [generate_measures_infection]
    outputs:
       moderately_sensitive:
        plot1: output/consult_age_1.jpeg
        plot2: output/consult_age_2.jpeg
        plot3: output/consult_all.jpeg
        csv1: output/consultation_rate.csv
        csv2: output/consultation_GP_rate.csv

  describe_infection_prescribed_percent:
    run: r:latest analysis/plot/infection_prescibed_percent.R
    needs: [generate_measures_infection]
    outputs:
       moderately_sensitive:
        plot1: output/infection_ab_precent_p1.jpeg
        plot2: output/infection_ab_precent_p2.jpeg
        plot3: output/infection_ab_precent_i1.jpeg
        plot4: output/infection_ab_precent_i2.jpeg
        plot5: output/infection_ab_precent_all.jpeg
        csv1: output/prescribed_infection_prevalent.csv
        csv2: output/prescribed_infection_incident.csv

  describe_top10ABtypes_byInfection:
    run: r:latest analysis/plot/abtypes_top10.R
    needs: [generate_measures_infection]
    outputs:
       moderately_sensitive:
        plot1: output/abtype_UTI.jpeg
        plot2: output/abtype_URTI.jpeg
        plot3: output/abtype_LRTI.jpeg
        plot4: output/abtype_sinusitis.jpeg
        plot5: output/abtype_ot_externa.jpeg
        plot6: output/abtype_otmedia.jpeg
        plot7: output/abtype_percent_UTI.jpeg
        plot8: output/abtype_percent_URTI.jpeg
        plot9: output/abtype_percent_LRTI.jpeg
        plot10: output/abtype_percent_sinusitis.jpeg
        plot11: output/abtype_percent_ot_externa.jpeg
        plot12: output/abtype_percent_otmedia.jpeg
        csv: output/abtype_top10_by_infection.csv
  
  # describe_top10ABtypes_total:
  #   run: r:latest analysis/plot/types_ab_prescriptions.R
  #   needs: [generate_study_population]
  #   outputs:
  #      moderately_sensitive:
  #       plot1: output/abtype_all_Rx.jpeg
  #       plot2: output/abtype_all_Rx_percent.jpeg

  describe_broad_proportion:
    run: r:latest analysis/plot/broad_proportion.R
    needs: [generate_measures]
    outputs:
       moderately_sensitive:
        plot1: output/broad_proportions_line_age.jpeg
        plot2: output/broad_proportions_line.jpeg
        csv1: output/broad_prescriptions_proportion.csv

  describe_same_day_ab_sgss:
    run: r:latest analysis/plot/same_day_covid_ab_sgss.R
    needs: [generate_measures]
    outputs:
       moderately_sensitive:
        plot1: output/same_day_ab_prop_line_sgss_age.jpeg
        plot2: output/same_day_ab_prop_line_sgss.jpeg
        csv1: output/same_day_ab_prop_sgss.csv

  describe_same_day_ab_gp:
    run: r:latest analysis/plot/same_day_covid_ab_gp.R
    needs: [generate_measures]
    outputs:
       moderately_sensitive:
        plot1: output/same_day_ab_prop_line_gp_age.jpeg
        plot2: output/same_day_ab_prop_line_gp.jpeg
        csv1: output/same_day_ab_prop_gp.csv

  generate_study_population_antibiotics: #  Antibiotics_recorded_01
    run: cohortextractor:latest generate_cohort --study-definition study_definition_antibiotics --index-date-range "2019-01-01 to today" --skip-existing --output-dir=output/measures --output-format=csv.gz
    outputs:
      highly_sensitive:
        cohort: output/measures/input_antibiotics_*.csv.gz

  describe_ab_recoded_indication_process: #  Antibiotics_recorded_02: generate prescription-level dataset
    run: r:latest analysis/process/ab_recorded_indication_transform.R
    needs: [generate_study_population_antibiotics]
    outputs:
       highly_sensitive:
        rds1: output/measures/recorded_ab_2019.rds
        rds2: output/measures/recorded_ab_2020.rds
        rds3: output/measures/recorded_ab_2021.rds
        rds4: output/measures/recorded_ab_2022.rds

Timeline

  • Created:

  • Started:

  • Finished:

  • Runtime: 36:34:08

These timestamps are generated and stored using the UTC timezone on the TPP backend.

Job information

Status
Succeeded
Backend
TPP
Workspace
service_eval_work
Requested by
Ya-Ting Yang
Branch
main
Force run dependencies
No
Git commit hash
ed2449d
Requested actions
  • generate_study_population_antibiotics
  • describe_ab_recoded_indication_process

Code comparison

Compare the code used in this Job Request