Skip to content

Job request: 2888

Organisation:
Bennett Institute
Workspace:
comparative-ve-research_main
ID:
qammykcrul324y2o
View repo View project.yaml

This page shows the technical details of what happened when authorised researcher Will Hulme requested one or more actions to be run against real patient data in the project, within a secure environment.

By cross-referencing the indicated Requested Actions with the Pipeline section below, you can infer what security level various outputs were written to. Outputs marked as highly_sensitive can never be viewed directly by a researcher; they can only request that code runs against them. Outputs marked as moderately_sensitive can be viewed by an approved researcher by logging into a highly secure environment. Only outputs marked as moderately_sensitive can be requested for release to the public, via a controlled output review service.

Jobs

Pipeline

Show project.yaml 
version: '3.0'

expectations:

  population_size: 100000

actions:

  hcw_extract:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_hcw
      --output-format feather
    outputs:
      highly_sensitive:
        cohort: output/input_hcw.feather

  hcw_process:
    run: r:latest analysis/hcw/process.R
    needs:
    - hcw_extract
    outputs:
      highly_sensitive:
        data: output/data/hcw_data_processed.rds
        datavax: output/data/hcw_data_vax.rds

  hcw_properties:
    run: r:latest analysis/process/data_properties.R output/data/hcw_data_processed.rds
      output/data_properties
    needs:
    - hcw_process
    outputs:
      moderately_sensitive:
        cohort: output/data_properties/hcw_data_processed*.txt

  hcw_descr:
    run: r:latest analysis/hcw/descr.R
    needs:
    - hcw_process
    outputs:
      moderately_sensitive:
        png: output/hcw/*.png
        svg: output/hcw/*.svg
        html: output/hcw/*.html
        csv: output/hcw/*.csv

  design:
    run: r:latest analysis/process/design.R
    outputs:
      moderately_sensitive:
        metadata: output/data/metadata*

  extract:
    run: cohortextractor:latest generate_cohort --study-definition study_definition
      --output-format feather
    outputs:
      highly_sensitive:
        cohort: output/input.feather

  data_process:
    run: r:latest analysis/process/data_process.R
    needs:
    - extract
    outputs:
      highly_sensitive:
        cohort: output/data/data_processed.rds

  data_properties:
    run: r:latest analysis/process/data_properties.R output/data/data_processed.rds
      output/data_properties
    needs:
    - data_process
    outputs:
      highly_sensitive:
        cohort: output/data_properties/data_processed*.txt

  data_selection:
    run: r:latest analysis/process/data_selection.R
    needs:
    - data_process
    outputs:
      highly_sensitive:
        data_allvax: output/data/data_cohort_allvax.rds
        data: output/data/data_cohort.rds
      moderately_sensitive:
        flow: output/data/flowchart.csv

  descr_table1:
    run: r:latest analysis/descriptive/table1.R
    needs:
    - design
    - data_selection
    outputs:
      moderately_sensitive:
        html: output/descriptive/tables/table1*.html
        csv: output/descriptive/tables/table1*.csv

  descr_table1_allvax:
    run: r:latest analysis/descriptive/table1_allvax.R
    needs:
    - design
    - data_selection
    outputs:
      moderately_sensitive:
        html: output/descriptive/tables/table1_allvax.html
        csv: output/descriptive/tables/table1_allvax.csv

  descr_irr:
    run: r:latest analysis/descriptive/table_irr.R output/data/data_processed.rds
      output/data_properties
    needs:
    - design
    - data_selection
    outputs:
      moderately_sensitive:
        html: output/descriptive/tables/table_irr.html
        csv: output/descriptive/tables/table_irr.csv

  descr_km:
    run: r:latest analysis/descriptive/km.R output/data/data_processed.rds output/data_properties
    needs:
    - design
    - data_selection
    outputs:
      moderately_sensitive:
        png: output/descriptive/km/plot_survival*.png
        svg: output/descriptive/km/plot_survival*.svg

  model_test_timesincevax:
    run: r:latest analysis/models/model_cox.R test timesincevax
    needs:
    - design
    - data_selection
    outputs:
      highly_sensitive:
        data: output/test/timesincevax/data_cox_split.rds
        models: output/test/timesincevax/model*.rds
      moderately_sensitive:
        logs: output/test/timesincevax/log*.txt
        glance: output/test/timesincevax/glance*.csv

  report_test_timesincevax:
    run: r:latest analysis/models/report_cox.R test timesincevax
    needs:
    - design
    - model_test_timesincevax
    outputs:
      moderately_sensitive:
        svg: output/test/timesincevax/forest*.svg
        png: output/test/timesincevax/forest*.png
        csv: output/test/timesincevax/estimates*.csv

  model_test_calendar:
    run: r:latest analysis/models/model_cox.R test calendar
    needs:
    - design
    - data_selection
    outputs:
      highly_sensitive:
        data: output/test/calendar/data_cox_split.rds
        models: output/test/calendar/model*.rds
      moderately_sensitive:
        logs: output/test/calendar/log*.txt
        glance: output/test/calendar/glance*.csv

  report_test_calendar:
    run: r:latest analysis/models/report_cox.R test calendar
    needs:
    - design
    - model_test_calendar
    outputs:
      moderately_sensitive:
        svg: output/test/calendar/forest*.svg
        png: output/test/calendar/forest*.png
        csv: output/test/calendar/estimates*.csv

  model_postest_timesincevax:
    run: r:latest analysis/models/model_cox.R postest timesincevax
    needs:
    - design
    - data_selection
    outputs:
      highly_sensitive:
        data: output/postest/timesincevax/data_cox_split.rds
        models: output/postest/timesincevax/model*.rds
      moderately_sensitive:
        logs: output/postest/timesincevax/log*.txt
        glance: output/postest/timesincevax/glance*.csv

  report_postest_timesincevax:
    run: r:latest analysis/models/report_cox.R postest timesincevax
    needs:
    - design
    - model_postest_timesincevax
    outputs:
      moderately_sensitive:
        svg: output/postest/timesincevax/forest*.svg
        png: output/postest/timesincevax/forest*.png
        csv: output/postest/timesincevax/estimates*.csv

  model_postest_calendar:
    run: r:latest analysis/models/model_cox.R postest calendar
    needs:
    - design
    - data_selection
    outputs:
      highly_sensitive:
        data: output/postest/calendar/data_cox_split.rds
        models: output/postest/calendar/model*.rds
      moderately_sensitive:
        logs: output/postest/calendar/log*.txt
        glance: output/postest/calendar/glance*.csv

  report_postest_calendar:
    run: r:latest analysis/models/report_cox.R postest calendar
    needs:
    - design
    - model_postest_calendar
    outputs:
      moderately_sensitive:
        svg: output/postest/calendar/forest*.svg
        png: output/postest/calendar/forest*.png
        csv: output/postest/calendar/estimates*.csv

  model_emergency_timesincevax:
    run: r:latest analysis/models/model_cox.R emergency timesincevax
    needs:
    - design
    - data_selection
    outputs:
      highly_sensitive:
        data: output/emergency/timesincevax/data_cox_split.rds
        models: output/emergency/timesincevax/model*.rds
      moderately_sensitive:
        logs: output/emergency/timesincevax/log*.txt
        glance: output/emergency/timesincevax/glance*.csv

  report_emergency_timesincevax:
    run: r:latest analysis/models/report_cox.R emergency timesincevax
    needs:
    - design
    - model_emergency_timesincevax
    outputs:
      moderately_sensitive:
        svg: output/emergency/timesincevax/forest*.svg
        png: output/emergency/timesincevax/forest*.png
        csv: output/emergency/timesincevax/estimates*.csv

  model_emergency_calendar:
    run: r:latest analysis/models/model_cox.R emergency calendar
    needs:
    - design
    - data_selection
    outputs:
      highly_sensitive:
        data: output/emergency/calendar/data_cox_split.rds
        models: output/emergency/calendar/model*.rds
      moderately_sensitive:
        logs: output/emergency/calendar/log*.txt
        glance: output/emergency/calendar/glance*.csv

  report_emergency_calendar:
    run: r:latest analysis/models/report_cox.R emergency calendar
    needs:
    - design
    - model_emergency_calendar
    outputs:
      moderately_sensitive:
        svg: output/emergency/calendar/forest*.svg
        png: output/emergency/calendar/forest*.png
        csv: output/emergency/calendar/estimates*.csv

  model_covidadmitted_timesincevax:
    run: r:latest analysis/models/model_cox.R covidadmitted timesincevax
    needs:
    - design
    - data_selection
    outputs:
      highly_sensitive:
        data: output/covidadmitted/timesincevax/data_cox_split.rds
        models: output/covidadmitted/timesincevax/model*.rds
      moderately_sensitive:
        logs: output/covidadmitted/timesincevax/log*.txt
        glance: output/covidadmitted/timesincevax/glance*.csv

  report_covidadmitted_timesincevax:
    run: r:latest analysis/models/report_cox.R covidadmitted timesincevax
    needs:
    - design
    - model_covidadmitted_timesincevax
    outputs:
      moderately_sensitive:
        svg: output/covidadmitted/timesincevax/forest*.svg
        png: output/covidadmitted/timesincevax/forest*.png
        csv: output/covidadmitted/timesincevax/estimates*.csv

  model_covidadmitted_calendar:
    run: r:latest analysis/models/model_cox.R covidadmitted calendar
    needs:
    - design
    - data_selection
    outputs:
      highly_sensitive:
        data: output/covidadmitted/calendar/data_cox_split.rds
        models: output/covidadmitted/calendar/model*.rds
      moderately_sensitive:
        logs: output/covidadmitted/calendar/log*.txt
        glance: output/covidadmitted/calendar/glance*.csv

  report_covidadmitted_calendar:
    run: r:latest analysis/models/report_cox.R covidadmitted calendar
    needs:
    - design
    - model_covidadmitted_calendar
    outputs:
      moderately_sensitive:
        svg: output/covidadmitted/calendar/forest*.svg
        png: output/covidadmitted/calendar/forest*.png
        csv: output/covidadmitted/calendar/estimates*.csv

Timeline

  • Created:

  • Started:

  • Finished:

  • Runtime: 10:15:47

These timestamps are generated and stored using the UTC timezone on the TPP backend.

Job information

Status
Failed
Backend
TPP
Workspace
comparative-ve-research_main
Requested by
Will Hulme
Branch
main
Force run dependencies
Yes
Git commit hash
fdacaf7
Requested actions
  • run_all

Code comparison

Compare the code used in this Job Request