Job request: 8070

  • ID: ob2tx53kubeyolkw
  • Workspace: cc_2
View Repo View project.yaml

This page shows the technical details of what happened when authorised researcher rriefu requested one or more actions to be run against real patient data in the BRIT Antibiotic Research project, within a secure environment.

By cross-referencing the indicated Requested Actions with the Pipeline section below, you can infer what security level various outputs were written to. Outputs marked as highly_sensitive can never be viewed directly by a researcher; they can only request that code runs against them. Outputs marked as moderately_sensitive can be viewed by an approved researcher by logging into a highly secure environment. Only outputs marked as moderately_sensitive can be requested for release to the public, via a controlled output review service.

Jobs

ID Status Action
u3qass3nie7e3ypo succeeded model

Pipeline

Show Hide project.yaml
version: '3.0'

expectations:
  population_size: 1000

actions:
  

  generate_study_population_covid_primarycare:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_covid_primarycare
    outputs:
      highly_sensitive:
        cohort: output/input_covid_primarycare.csv
  
  generate_study_population_covid_SGSS:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_covid_SGSS
    outputs:
      highly_sensitive:
        cohort: output/input_covid_SGSS.csv

  generate_study_population_covid_admission:
    run: cohortextractor:latest generate_cohort --study-definition study_definition_covid_admission
    outputs:
      highly_sensitive:
        cohort: output/input_covid_admission.csv

  process_1:  # Covid ons death
    run: r:latest analysis/process_1.R
    needs: [generate_study_population_covid_primarycare, generate_study_population_covid_SGSS,generate_study_population_covid_admission]
    outputs:
      highly_sensitive:
        case: output/case_covid_hosp.csv 
        control: output/control_covid_infection.csv 

  matching: #R MatchIt  matching with replacement
    run: r:latest -e 'rmarkdown::render("analysis/matching.Rmd", knit_root_dir = "/workspace", output_dir="/workspace/output")'
    needs: [process_1]
    outputs:
      moderately_sensitive:
        html: output/matching.html
      highly_sensitive: 
        rds1: output/matched_patients.rds
        rds2: output/unmatched_cases.rds
        csv: output/matched_patients_id.csv

  extract_variables: 
    run: cohortextractor:latest generate_cohort --study-definition study_definition_outcome
    needs: [matching]
    outputs:
      highly_sensitive:
        cohort: output/input_outcome.csv

  process_Rmatching: # add variables 
    run: r:latest analysis/process_Rmatching.R
    needs: [extract_variables,matching]
    outputs:
      highly_sensitive:
        cohort1: output/matched_outcome.rds
        cohort2: output/matched_outcome_check.rds # filter died $ de-regist again
        rds1: output/abtype79.rds
        rds2: output/comor17.rds

  table1: # compare before & after matching 
    run: r:latest -e 'rmarkdown::render("analysis/table1.Rmd", knit_root_dir = "/workspace", output_dir="/workspace/output")'
    needs: [process_1,process_Rmatching]
    outputs:
      moderately_sensitive:
        html: output/table1.html

  table2: # confounders
    run: r:latest -e 'rmarkdown::render("analysis/table2.Rmd", knit_root_dir = "/workspace", output_dir="/workspace/output")'
    needs: [process_Rmatching]
    outputs:
      moderately_sensitive:
        html: output/table2.html

  table2_ab: # ab
    run: r:latest -e 'rmarkdown::render("analysis/table2_ab.Rmd", knit_root_dir = "/workspace", output_dir="/workspace/output")'
    needs: [process_Rmatching]
    outputs:
      moderately_sensitive:
        html: output/table2_ab.html

  model: 
    run: r:latest -e 'rmarkdown::render("analysis/model.Rmd", knit_root_dir = "/workspace", output_dir="/workspace/output")'
    needs: [process_Rmatching]
    outputs:
      moderately_sensitive:
        html: output/model.html
        ################# check ok###########

  process_2: # died any cause
    run: r:latest analysis/process_2.R
    needs: [generate_study_population_covid_primarycare, generate_study_population_covid_SGSS]
    outputs:
      highly_sensitive:
        control: output/control_covid_infection_2.csv

  check_input: # 
    run: r:latest -e 'rmarkdown::render("analysis/check_input.Rmd", knit_root_dir = "/workspace", output_dir = "output")'
    needs: [process_Rmatching,process_2,extract_variables,matching,process_matching,extract_variables_outcome_2_2,extract_variables_outcome_2_4,extract_variables_outcome_2_6,matching_2,process_1,generate_study_population_covid_primarycare, generate_study_population_covid_SGSS]
    outputs:
      moderately_sensitive:
        html: output/check_input.html
        
# OS matching without replacement, ratio 2-4-6
  matching_2:
    run: python:latest python analysis/matching_outcome_2.py
    needs: [process_1]
    outputs:
      moderately_sensitive: 
        matching_report2: output/matching_report_infection_hosp_2.txt
        matching_report4: output/matching_report_infection_hosp_4.txt
        matching_report6: output/matching_report_infection_hosp_6.txt
      highly_sensitive: 
        combined2: output/matched_combined_infection_hosp_2.csv
        combined4: output/matched_combined_infection_hosp_4.csv
        combined6: output/matched_combined_infection_hosp_6.csv

  extract_variables_outcome_2_2: 
    run: cohortextractor:latest generate_cohort --study-definition study_definition_outcome_2_2
    needs: [matching_2]
    outputs:
      highly_sensitive:
        cohort: output/input_outcome_2_2.csv
    
  extract_variables_outcome_2_4: 
    run: cohortextractor:latest generate_cohort --study-definition study_definition_outcome_2_4
    needs: [matching_2]
    outputs:
      highly_sensitive:
        cohort: output/input_outcome_2_4.csv
  
  extract_variables_outcome_2_6: 
    run: cohortextractor:latest generate_cohort --study-definition study_definition_outcome_2_6
    needs: [matching_2]
    outputs:
      highly_sensitive:
        cohort: output/input_outcome_2_6.csv

  check_matching_2:
    run: r:latest -e 'rmarkdown::render("analysis/check_matching_2.Rmd", knit_root_dir = "/workspace", output_dir="/workspace/output")'
    needs: [matching_2]
    outputs:
      moderately_sensitive:
        html: output/check_matching_2.html

  process_matching: # add variables 
    run: r:latest analysis/process_matching.R
    needs: [extract_variables_outcome_2_6,extract_variables_outcome_2_4,extract_variables_outcome_2_2,matching_2]
    outputs:
      highly_sensitive:
        cohort1: output/matched_outcome_2.csv
        cohort2: output/matched_outcome_4.csv
        cohort3: output/matched_outcome_6.csv

  check_exposures_6:
    run: r:latest -e 'rmarkdown::render("analysis/check_exposures_6.Rmd", knit_root_dir = "/workspace", output_dir="/workspace/output")'
    needs: [process_matching]
    outputs:
      moderately_sensitive:
        html: output/check_exposures_6.html 

  check_exposures_4:
    run: r:latest -e 'rmarkdown::render("analysis/check_exposures_4.Rmd", knit_root_dir = "/workspace", output_dir="/workspace/output")'
    needs: [process_matching]
    outputs:
      moderately_sensitive:
        html: output/check_exposures_4.html 

  check_exposures_2:
    run: r:latest -e 'rmarkdown::render("analysis/check_exposures_2.Rmd", knit_root_dir = "/workspace", output_dir="/workspace/output")'
    needs: [process_matching]
    outputs:
      moderately_sensitive:
        html: output/check_exposures_2.html 

# # R MatchIt

#   matching: # matching with replacement
#     run: r:latest -e 'rmarkdown::render("analysis/matching.Rmd", knit_root_dir = "/workspace", output_dir="/workspace/output")'
#     needs: [process_1]
#     outputs:
#       moderately_sensitive:
#         html: output/matching.html
#       highly_sensitive: 
#         rds1: output/matched_patients.rds
#         rds2: output/unmatched_patients.rds
#         csv: output/matched_patients_id.csv

  check_matching:
    run: r:latest -e 'rmarkdown::render("analysis/check_matching.Rmd", knit_root_dir = "/workspace", output_dir="/workspace/output")'
    needs: [matching]
    outputs:
      moderately_sensitive:
        html: output/check_matching.html

  # extract_variables: 
  #   run: cohortextractor:latest generate_cohort --study-definition study_definition_outcome
  #   needs: [matching]
  #   outputs:
  #     highly_sensitive:
  #       cohort: output/input_outcome.csv
  
  # process_Rmatching: # add variables 
  #   run: r:latest analysis/process_Rmatching.R
  #   needs: [extract_variables,matching]
  #   outputs:
  #     highly_sensitive:
  #       cohort1: output/matched_outcome.rds
  #       rds1: output/abtype79.rds
  #       rds2: output/comor17.rds

  check_exposures_cont: # continuous variables
    run: r:latest -e 'rmarkdown::render("analysis/check_exposures_cont.Rmd", knit_root_dir = "/workspace", output_dir="/workspace/output")'
    needs: [process_Rmatching]
    outputs:
      moderately_sensitive:
        html: output/check_exposures_cont.html
  
  check_exposures_cat: # category variables
    run: r:latest -e 'rmarkdown::render("analysis/check_exposures_cat.Rmd", knit_root_dir = "/workspace", output_dir="/workspace/output")'
    needs: [process_Rmatching]
    outputs:
      moderately_sensitive:
        html: output/check_exposures_cat.html

  modeling: # ab level
    run: r:latest -e 'rmarkdown::render("analysis/modeling.Rmd", knit_root_dir = "/workspace", output_dir="/workspace/output")'
    needs: [process_Rmatching]
    outputs:
      moderately_sensitive:
        html: output/modeling.html

  modeling2: # ab level+ CCI + covid vaccine
    run: r:latest -e 'rmarkdown::render("analysis/modeling2.Rmd", knit_root_dir = "/workspace", output_dir="/workspace/output")'
    needs: [process_Rmatching]
    outputs:
      moderately_sensitive:
        html: output/modeling2.html

  modeling3: # all
    run: r:latest -e 'rmarkdown::render("analysis/modeling3.Rmd", knit_root_dir = "/workspace", output_dir="/workspace/output")'
    needs: [process_Rmatching]
    outputs:
      moderately_sensitive:
        html: output/modeling3.html

  describe_ab: 
    run: r:latest -e 'rmarkdown::render("analysis/describe_ab.Rmd", knit_root_dir = "/workspace", output_dir="/workspace/output")'
    needs: [process_Rmatching]
    outputs:
      moderately_sensitive:
        html: output/describe_ab.html
  
  # abtype modeling
  modeling4: # all
    run: r:latest -e 'rmarkdown::render("analysis/modeling4.Rmd", knit_root_dir = "/workspace", output_dir="/workspace/output")'
    needs: [process_Rmatching]
    outputs:
      moderately_sensitive:
        html: output/modeling4.html


    # broad ab modeling
  modeling5: # all
    run: r:latest -e 'rmarkdown::render("analysis/modeling5.Rmd", knit_root_dir = "/workspace", output_dir="/workspace/output")'
    needs: [process_Rmatching]
    outputs:
      moderately_sensitive:
        html: output/modeling5.html



# unmatched controls
  extract_variables_unmatched: 
    run: cohortextractor:latest generate_cohort --study-definition study_definition_unmatched
    needs: [process_1]
    outputs:
      highly_sensitive:
        cohort: output/input_unmatched.csv
  
  process_unmatched: # add variables 
    run: r:latest analysis/process_unmatched.R
    needs: [extract_variables_unmatched]
    outputs:
      highly_sensitive:
        cohort1: output/unmatched_outcome.rds

  # check_exposures_cont_unmatched: # continuous variables
  #   run: r:latest -e 'rmarkdown::render("analysis/check_exposures_cont_unmatched.Rmd", knit_root_dir = "/workspace", output_dir="/workspace/output")'
  #   needs: [process_unmatched]
  #   outputs:
  #     moderately_sensitive:
  #       html: output/check_exposures_cont_unmatched.html
  
  check_exposures_cat_unmatched: # category variables
    run: r:latest -e 'rmarkdown::render("analysis/check_exposures_cat_unmatched.Rmd", knit_root_dir = "/workspace", output_dir="/workspace/output")'
    needs: [process_unmatched]
    outputs:
      moderately_sensitive:
        html: output/check_exposures_cat_unmatched.html

State

State is inferred from the related Jobs.

Status: Succeeded

Timings

Timings set to UTC timezone.

  • Created:
  • Started:
  • Finished:
  • Runtime: 00:02:43

Config

  • Backend:
    TPP
  • Workspace:
    cc_2
  • Branch:
    CC_outcome_2
  • Creator:
    rriefu
  • Force run dependencies:
    False
  • Git Commit Hash:
    1c6bd1a
  • Requested actions:
    • model