ProSEAlgorithm.h

Go to the documentation of this file.

// Copyright (c) 2002-present, The OpenMS Team -- EKU Tuebingen, ETH Zurich, and FU Berlin
// SPDX-License-Identifier: BSD-3-Clause
//
// --------------------------------------------------------------------------
// $Maintainer:  $
// $Authors: Raphael Förster $
// --------------------------------------------------------------------------
 
#pragma once
 
#include <OpenMS/CONCEPT/ProgressLogger.h>
#include <OpenMS/DATASTRUCTURES/DefaultParamHandler.h>
 
#include <OpenMS/ANALYSIS/ID/FragmentIndex.h>
#include <OpenMS/ANALYSIS/ID/FragmentIonLikelihoodModel.h>
#include <OpenMS/ANALYSIS/ID/OpenSearchModificationAnalysis.h>
#include <OpenMS/CHEMISTRY/EnzymaticDigestion.h>
#include <OpenMS/CHEMISTRY/ModifiedPeptideGenerator.h>
#include <OpenMS/FORMAT/FASTAFile.h>
#include <OpenMS/KERNEL/MSExperiment.h>
#include <OpenMS/METADATA/PeptideIdentificationList.h>
 
#include <algorithm>   // std::min (used by inline computeModMatchTolerance_)
#include <functional>  // std::hash (ProteinMapping_), std::function (FileResultConsumer, loadSpectra)
#include <iosfwd>      // std::ostream (renderRunSummary / renderModificationSummary)
#include <limits>
#include <map>
#include <memory>      // std::shared_ptr (released_hits_)
#include <string>      // std::string (renderRunSummaryJson return / manifest)
#include <string_view>
#include <unordered_map>
#include <unordered_set>
#include <utility>     // std::pair (renderRunSummaryJson manifest)
#include <vector>
 
namespace OpenMS
{
 
class Percolator;
class TheoreticalSpectrumGenerator;
 
class OPENMS_DLLAPI ProSEAlgorithm :
  public DefaultParamHandler,
  public ProgressLogger
{
  public:
    ProSEAlgorithm(); 
 
    static constexpr const char* CHIMERIC_RANK = "chimeric_rank";
 
    static constexpr double MIN_DECOYS_PER_TARGET = 0.5;
    static constexpr double MAX_DECOYS_PER_TARGET = 2.0;
 
    static constexpr const char* INPUT_PRECURSOR_TOLERANCES = "input_precursor_mass_tolerances";
    static constexpr const char* INPUT_FRAGMENT_TOLERANCES = "input_fragment_mass_tolerances";
 
    enum class ExitCodes
    {
      EXECUTION_OK,
      INPUT_FILE_EMPTY,
      UNEXPECTED_RESULT,
      UNKNOWN_ERROR,
      ILLEGAL_PARAMETERS
    };
 
    struct RunStatistics
    {
      std::string input_file;                  
      Size ms2_spectra = 0;                    
      Size profile_spectra = 0;                
      Size charge_out_of_range_spectra = 0;    
      Size matched_spectra = 0;                
      Size target_psms = 0;                    
      Size decoy_psms = 0;                     
      bool fdr_applied = false;                
      double achieved_psm_fdr = -1.0;          
      Size unique_peptides = 0;                
      Size unique_proteins = 0;                
      std::map<Int, Size> charge_histogram;    
      std::map<Size, Size> missed_cleavage_histogram; 
      bool score_stats_valid = false;          
      double hyperscore_min = 0.0;
      double hyperscore_median = 0.0;
      double hyperscore_max = 0.0;
      bool prec_tol_valid = false;             
      double prec_err_median = 0.0, prec_err_mad = 0.0, prec_err_recommended = 0.0;
      bool frag_tol_valid = false;             
      double frag_err_median = 0.0, frag_err_mad = 0.0, frag_err_recommended = 0.0;
      double seconds_search = 0.0;             
      double seconds_calibration = 0.0;        
      double seconds_fdr = 0.0;                
      std::vector<std::pair<std::string, DataValue>> recalibration;
    };
 
    struct SharedSearchStats
    {
      std::string database_file;               
      std::string enzyme;
      double precursor_tol_lower = 0.0, precursor_tol_upper = 0.0;
      std::string precursor_tol_unit;
      double fragment_tol = 0.0;
      std::string fragment_tol_unit;
      Int min_charge = 0, max_charge = 0;
      Size missed_cleavages = 0;
      std::vector<std::string> fixed_mods, variable_mods, ion_series;
      bool open_search = false;
      bool calibration_enabled = false;
      std::string recalibrate;                 
      bool peptide_mass_mode = false;
      bool chunked = false;
      std::string decoy_mode;                  
      double psm_fdr_threshold = 0.0, protein_fdr_threshold = 0.0;
      Size db_target_proteins = 0;             
      Size db_decoy_proteins = 0;              
      Size indexed_peptides = 0;               
      Size indexed_fragments = 0;              
      double seconds_index_build = 0.0;        
      double seconds_total = 0.0;              
      double seconds_output = 0.0;             
      double seconds_qpx_extended = 0.0;       
    };
 
    struct PercolatorRescoring
    {
      bool applied = false;
      std::string skipped_reason;
      std::string warning;
      std::string model;
    };
 
    struct PercolatorTraining
    {
      int subset_max_train = 0;  
      std::string subset_mode = "out_of_fold"; 
      int curve_start = 100000;                
      double curve_min_gain = 0.002;           
      int iterations = 10;       
      double tron_epsilon = 1e-7;  
      bool warm_start = false;     
      bool single_cneg = false;    
      std::string solver = "newton"; 
      std::string use_pi0 = "true"; 
      bool calc_mass_feature = false; 
      bool fixed_enzyme_termini = false; 
      bool peptide_level = true;   
    };
 
    static void stratumQValues(PeptideIdentificationList& peptide_ids);
 
    static bool reuseTrainingDraw(UInt64 seed, Size file, Size spectrum, double fraction);
 
    static bool reuseCalibrationDraw(UInt64 seed, Size file, Size spectrum, double fraction);
 
    struct SearchResult
    {
      ExitCodes exit_code = ExitCodes::EXECUTION_OK;
      std::vector<ProteinIdentification> protein_ids;
      PeptideIdentificationList peptide_ids;
      OpenSearchModificationAnalysis::OpenSearchAnalysisResult modification_analysis;
      bool is_open_search = false;
      RunStatistics stats;
      PercolatorRescoring percolator;
      PeptideIdentificationList pre_fdr_peptide_ids; 
      PeptideIdentificationList inference_only_peptide_ids;
    };
 
    struct MultiFileSearchResult
    {
      std::vector<SearchResult> per_file;
      SearchResult aggregate;
 
      std::string decoy_string;
      bool decoy_is_prefix = true;
      bool have_decoys = false;
 
      SharedSearchStats shared;
    };
 
    struct MergedResult
    {
      std::vector<ProteinIdentification> protein_ids; 
      PeptideIdentificationList peptide_ids;
      bool protein_fdr_applied = false;
      std::vector<std::unordered_map<std::string, double>> file_peptide_q_values;
    };
 
    struct SearchContext
    {
      std::vector<FASTAFile::FASTAEntry> db;
      FragmentIndex fragment_index;
 
      bool release_fragment_index_after_scoring = false;
 
      std::string decoy_string;
      bool decoy_is_prefix = true;
      bool have_decoys = false;
      bool electron_ions = false;
    };
 
    ExitCodes search(const std::string& in_spectra,
      const std::string& in_db,
      std::vector<ProteinIdentification>& prot_ids,
      PeptideIdentificationList& pep_ids) const;
 
    static bool applyCompleteSetProteinFDR(std::vector<ProteinIdentification>& protein_ids,
                                           PeptideIdentificationList& peptide_ids,
                                           const std::string& decoy_string,
                                           bool decoy_is_prefix,
                                           double protein_fdr,
                                           bool remove_decoys,
                                           const PeptideIdentificationList& inference_only_psms);
 
    static std::string inferProteins(std::vector<ProteinIdentification>& protein_ids,
                                     const PeptideIdentificationList& peptide_ids,
                                     const PeptideIdentificationList& inference_only_psms = PeptideIdentificationList());
 
    static void removeUnreportedProteins(std::vector<ProteinIdentification>& protein_ids,
                                         const PeptideIdentificationList& peptide_ids,
                                         const PeptideIdentificationList& also_referencing = PeptideIdentificationList());
 
    static Size removeDuplicateProteins(std::vector<FASTAFile::FASTAEntry>& db, const std::string& source);
 
    void annotatePsmQValues(PeptideIdentificationList& peptide_ids) const;
 
    static bool annotatePeptideQValues(PeptideIdentificationList& peptide_ids, Size* target_peptides_1pct = nullptr);
 
    static PercolatorRescoring rescoreWithPercolator(std::vector<ProteinIdentification>& protein_ids,
                                                     PeptideIdentificationList& peptide_ids,
                                                     int threads,
                                                     Size max_train_psms = 0);
 
    static std::string pinEnzymeName(const std::string& enzyme);
 
    static std::string pinFeatureEnzyme(const std::string& enzyme, bool fixed_enzyme_termini);
 
    static PeakMap loadSpectra(const std::string& filename, const std::function<bool(const MSSpectrum&)>& keep = {});
 
    bool applyDeferredPsmFDR(const std::vector<ProteinIdentification>& protein_ids,
                             PeptideIdentificationList& peptide_ids,
                             bool rescored,
                             double psm_fdr,
                             PeptideIdentificationList* inference_only_psms = nullptr) const;
 
    SearchResult searchWithModificationAnalysis(const std::string& in_spectra,
                                                const std::string& in_db,
                                                const std::string& output_base_name = "") const;
 
    ExitCodes search(PeakMap& spectra,
                     const std::vector<FASTAFile::FASTAEntry>& fasta_db,
                     std::vector<ProteinIdentification>& prot_ids,
                     PeptideIdentificationList& pep_ids) const;
 
    SearchContext prepareContext(const std::vector<FASTAFile::FASTAEntry>& fasta_db) const;
 
    SearchContext prepareContext(const std::vector<FASTAFile::FASTAEntry>& fasta_db, bool electron_ions) const;
 
    ExitCodes search(PeakMap& spectra,
                     SearchContext& ctx,
                     std::vector<ProteinIdentification>& prot_ids,
                     PeptideIdentificationList& pep_ids) const;
 
    SearchResult searchWithModificationAnalysis(PeakMap& spectra,
                                                const std::vector<FASTAFile::FASTAEntry>& fasta_db,
                                                const std::string& output_base_name = "") const;
 
    MultiFileSearchResult searchWithModificationAnalysis(
        const std::vector<std::string>& in_spectra_files,
        const std::vector<FASTAFile::FASTAEntry>& fasta_db,
        const std::vector<std::string>& output_base_names = {},
        const std::string& aggregate_base_name = "",
        bool build_pooled_aggregate = true) const;
 
    MultiFileSearchResult searchWithModificationAnalysis(
        const std::vector<std::string>& in_spectra_files,
        const std::string& in_db,
        const std::vector<std::string>& output_base_names = {},
        const std::string& aggregate_base_name = "",
        bool build_pooled_aggregate = true) const;
 
    MergedResult mergeFileResults(MultiFileSearchResult& result, const StringList& in_spectra_files,
                                  bool remove_decoys = true) const;
 
    static void splitMergedPsms(MergedResult& merged, MultiFileSearchResult& result);
 
    void setParameterNamePrefix(const std::string& prefix) { parameter_name_prefix_ = prefix; }
 
    void setKeepPreFdrPsms(bool keep)
    {
      keep_pre_fdr_psms_ = keep;
      if (!keep) { pre_fdr_peptide_ids_ = PeptideIdentificationList(); }
    }
 
    using FileResultConsumer = std::function<void(Size, const SearchResult&)>;
 
    void setFileResultConsumer(FileResultConsumer consumer)
    {
      file_result_consumer_ = std::move(consumer);
    }
 
  protected:
    void updateMembers_() override;
 
    struct AnnotatedHit_
    {
      AASequence sequence;
      /*
      std::string_view sequence;
      SignedSize peptide_mod_index; ///< enumeration index of the non-RNA peptide modification
      */
      // Layout: doubles first, then floats, then int, then uint16_t — minimizes padding (40 bytes excluding AASequence)
      double score = 0; 
      double delta_mass = 0.0; 
      float prefix_fraction = 0; 
      float suffix_fraction = 0; 
      float mean_error = 0.0f; 
      int isotope_error = 0; 
      uint16_t applied_charge = 0; 
      uint16_t matched_prefix_ions = 0; 
      uint16_t matched_suffix_ions = 0; 
      UInt32 index_peptide = 0; 
 
      static bool hasBetterScore(const AnnotatedHit_& a, const AnnotatedHit_& b)
      {
        if (a.score != b.score) return a.score > b.score;
        return a.sequence < b.sequence;
      }
    };
 
    struct CandidatePoolStats_
    {
      double sum = 0.0;         
      double sumsq = 0.0;       
      double best = 0.0;        
      double second_best = 0.0; 
      Size count = 0;           
      std::unordered_set<std::string> seen_candidates;
      int scoring_thread = 0;
 
      void add(double score)
      {
        if (score > best) { second_best = best; best = score; }
        else if (score > second_best) { second_best = score; }
        sum += score;
        sumsq += score * score;
        ++count;
      }
 
      double deltaScore() const { return best - second_best; }
 
      double zScore() const;
    };
 
    struct DeisotopingSettings_
    {
      unsigned int min_peaks{3};          
      bool charge_cap_precursor{false};   
      bool sum_intensity{false};          
    };
 
    static void preprocessSpectra_(PeakMap& exp,
                                   double fragment_mass_tolerance,
                                   bool fragment_mass_tolerance_unit_ppm,
                                   bool deisotope_requested,
                                   Size peaks_keep_n,
                                   Int peaks_window_top,
                                   const std::string& window_type,
                                   const DeisotopingSettings_& deisotoping,
                                   FragmentIonLikelihoodModel::PeakLists* ion_evidence = nullptr,
                                   bool ion_evidence_scored_peaks = false,
                                   PeakMap* evidence_spectra = nullptr,
                                   PeakMap* query_spectra = nullptr);
 
    static void preprocessSpectra_(PeakMap& exp,
                                   double fragment_mass_tolerance,
                                   bool fragment_mass_tolerance_unit_ppm,
                                   bool deisotope_requested,
                                   Size peaks_keep_n,
                                   Int peaks_window_top,
                                   const std::string& window_type = "auto");
 
    void annotateIonPriors_(const PeakMap& spectra,
                            const FragmentIonLikelihoodModel::PeakLists& evidence,
                            std::vector<ProteinIdentification>& protein_ids,
                            PeptideIdentificationList& peptide_ids) const;
 
    static AASequence reversedNoiseSequence_(const AASequence& sequence);
 
    static void filterLocalPeaks_(MSSpectrum& spectrum, Size peaks_per_window);
 
    enum class DecoyMode_
    {
      AUTO,     
      GENERATE, 
      IGNORE    
    };
 
    struct DecoyStrategy_
    {
      bool generate{false};       
      bool strip_existing{false}; 
      bool have_decoys{false};    
      std::string decoy_string;   
      bool is_prefix{true};       
      std::string strip_string;   
      bool strip_is_prefix{true}; 
    };
 
    DecoyStrategy_ resolveDecoyStrategy_(
        const std::vector<FASTAFile::FASTAEntry>& db) const;
 
    Param fragmentIndexParameters_(bool electron_ions = false) const;
 
    struct SpectrumGenerators_;
 
    struct MzRecalibration_;
 
    SpectrumGenerators_ spectrumGenerators_() const;
 
    struct LocalFragmentEvidence_
    {
      double chance_match_surprise = 0.0;
      double mass_competition_evidence = 0.0;
    };
 
    static std::vector<double> localPeakDensities_(const MSSpectrum& spectrum);
 
    static LocalFragmentEvidence_ localFragmentEvidence_(const MSSpectrum& spectrum,
                                                        const MSSpectrum& theoretical,
                                                        const std::vector<double>& densities,
                                                        double tolerance, bool ppm);
 
    static LocalFragmentEvidence_ localFragmentEvidence_(const MSSpectrum& spectrum,
                                                        const MSSpectrum& theoretical,
                                                        int max_charge,
                                                        const std::vector<double>& densities,
                                                        double tolerance, bool ppm,
                                                        std::vector<double>& alternatives);
 
    static LocalFragmentEvidence_ localFragmentEvidence_(const MSSpectrum& spectrum,
                                                        const MSSpectrum& theoretical,
                                                        const std::vector<Int>& charges,
                                                        int max_charge,
                                                        const std::vector<double>& densities,
                                                        double tolerance, bool ppm,
                                                        std::vector<double>& alternatives);
 
    static bool isElectronActivated_(const MSSpectrum& spectrum);
 
    Size countElectronActivated_(const PeakMap& spectra) const;
 
    std::vector<FASTAFile::FASTAEntry> buildDecoyAugmentedDB_(
        const std::vector<FASTAFile::FASTAEntry>& fasta_db,
        const DecoyStrategy_& strategy) const;
 
    std::vector<FASTAFile::FASTAEntry> buildDecoyAugmentedDB_(
        std::vector<FASTAFile::FASTAEntry>&& fasta_db,
        const DecoyStrategy_& strategy) const;
 
    MultiFileSearchResult searchWithModificationAnalysis_(
        const std::vector<std::string>& in_spectra_files,
        const std::vector<FASTAFile::FASTAEntry>& fasta_db_given,
        const std::string* in_db,
        const std::vector<std::string>& output_base_names,
        const std::string& aggregate_base_name,
        bool build_pooled_aggregate) const;
 
    void reportDecoyParity_(const SearchContext& ctx) const;
    SearchContext prepareContext_(std::vector<FASTAFile::FASTAEntry>&& fasta_db, bool electron_ions,
                                  const std::function<const PeakMap*(Size)>& searched_spectra = {}) const;
 
    bool restrictIndexToSpectra_() const { return !chimeric_enabled_ && !(calibration_enabled_ && calibration_recalibrate_ == "linear"); }
 
    void buildFragmentIndex_(SearchContext& ctx, const Param& params, const std::function<const PeakMap*(Size)>& searched_spectra) const;
 
    static double calibrationPassWindow_(double configured, bool ppm);
 
    static double measuredError_(double offset, double sigma);
 
    static double measuredCoverage_(const std::vector<double>& errors, bool fragment);
 
    bool chimericSearch_() const { return chimeric_enabled_ && database_chunk_size_ == 0; }
 
    struct CoIsolatedPSM_
    {
      Size parent = 0;             
      Precursor precursor;         
      std::vector<UInt32> removed; 
    };
 
    void searchCoIsolated_(const PeakMap& spectra,
                           const std::vector<std::vector<AnnotatedHit_>>& annotated_hits,
                           FragmentIndex& fi,
                           const std::vector<FASTAFile::FASTAEntry>& db,
                           const SpectrumGenerators_& generators,
                           double fragment_tol,
                           bool fragment_tol_ppm,
                           bool open_search_mode,
                           std::vector<CoIsolatedPSM_>& psms,
                           std::vector<std::vector<AnnotatedHit_>>& hits,
                           std::vector<CandidatePoolStats_>& pool_stats) const;
 
    static MSSpectrum residualSpectrum_(const MSSpectrum& spectrum, const std::vector<UInt32>& removed, const Precursor& precursor);
 
    std::vector<FASTAFile::FASTAEntry> buildCalibrationSample_(
        const std::vector<FASTAFile::FASTAEntry>& full_db) const;
 
    ExitCodes searchChunked_(
        PeakMap& spectra,
        std::vector<FASTAFile::FASTAEntry>& full_db,
        const DecoyStrategy_& strategy,
        std::vector<ProteinIdentification>& protein_ids,
        PeptideIdentificationList& peptide_ids) const;
 
    void scoreSpectraAgainstIndex_(
        const PeakMap& spectra,
        FragmentIndex& fi,
        const std::vector<FASTAFile::FASTAEntry>& db,
        const SpectrumGenerators_& generators,
        double effective_fragment_tol,
        bool fragment_mass_tolerance_unit_ppm,
        bool open_search_mode,
        std::vector<std::vector<AnnotatedHit_>>& annotated_hits,
        std::vector<CandidatePoolStats_>& pool_stats,
        const std::string& progress_label,
        const PeakMap* query_spectra = nullptr) const;
 
    static std::vector<SignedSize> scoringOrder_(const PeakMap& spectra, const PeakMap* query_spectra);
 
    void postProcessHits_(const PeakMap& exp,
      std::vector<std::vector<ProSEAlgorithm::AnnotatedHit_> >& annotated_hits,
      const std::vector<CandidatePoolStats_>& pool_stats,
      std::vector<ProteinIdentification>& protein_ids,
      PeptideIdentificationList& peptide_ids,
      Size top_hits,
      const StringList& modifications_fixed,
      const StringList& modifications_variable,
      Int peptide_missed_cleavages,
      double precursor_mass_tolerance,
      double fragment_mass_tolerance,
      const std::string& precursor_mass_tolerance_unit_ppm,
      const std::string& fragment_mass_tolerance_unit_ppm,
      const Int precursor_min_charge,
      const Int precursor_max_charge,
      const std::string& enzyme,
      const std::string& database_name,
      const PeakMap* evidence_spectra = nullptr,
      const std::vector<Size>* evidence_parent = nullptr,
      bool defer_peak_annotations = false,
      const MzRecalibration_* fragment_recalibration = nullptr) const;
 
    void annotateFragmentPeaks_(const PeakMap& exp, PeptideIdentificationList& peptide_ids,
                                double fragment_mass_tolerance, const std::string& fragment_mass_tolerance_unit_ppm,
                                const MzRecalibration_* fragment_recalibration = nullptr) const;
 
    void annotatePeptDeepFeatures_(const PeakMap& spectra,
                                   std::vector<ProteinIdentification>& protein_ids,
                                   PeptideIdentificationList& peptide_ids) const;
 
    bool peptdeep_enable_{false};
    std::string peptdeep_ms2_model_;
    std::string peptdeep_rt_model_;
    std::string peptdeep_instrument_{"QE"};
    double peptdeep_nce_{-1.0};
    std::string peptdeep_rt_model_type_{"b_spline"};
    int peptdeep_batch_size_{500};
 
    mutable double precursor_mass_tolerance_lower_{10.0};   
    mutable double precursor_mass_tolerance_upper_{10.0};   
    std::string precursor_mass_tolerance_unit_{"ppm"};
 
    Size precursor_min_charge_;
    Size precursor_max_charge_;
 
    double fragment_mass_tolerance_;
 
    std::string fragment_mass_tolerance_unit_;
 
    bool deisotope_requested_{true};
    bool scoring_multiple_charges_{false}; 
    int scoring_max_charge_{2};            
    bool mass_accuracy_score_{false};      
    double mass_error_sd_ppm_{7.0};        
 
    int scoringMaxCharge_(int precursor_charge) const
    {
      return scoring_multiple_charges_ ? std::max(1, std::min(precursor_charge - 1, scoring_max_charge_)) : 1;
    }
    Size peaks_keep_n_{0};     
    Int peaks_window_top_{20}; 
    std::string peaks_window_type_ {"auto"}; 
    DeisotopingSettings_ deisotoping_;        
    bool query_raw_spectrum_{false}; 
 
    StringList modifications_fixed_;
 
    StringList modifications_variable_;
 
    Size modifications_max_variable_mods_per_peptide_;
 
    std::string enzyme_;
 
    DecoyMode_ decoy_mode_{DecoyMode_::AUTO};
    std::string decoy_prefix_;
    bool decoy_keep_termini_ = false;   
    bool decoy_parity_report_ = false;  
    // The effective values of the current search: buildDecoyAugmentedDB_() (a const step) re-derives them from the
    // parameters at every database build and switches both off when the reused decoys rarely start with M (the
    // Met-share check of that function); nterm_options_disabled_ records that for the output's search parameters.
    mutable bool terminal_features_ = false;    
    mutable bool stratum_q_values_ = false;     
    mutable bool nterm_options_disabled_ = false; 
 
    bool rescore_{true}; 
    PercolatorTraining percolator_training_; 
    Param percolator_extra_; 
    mutable std::string reset_run_name_; 
    std::string reuse_mode_{"per_file"}; 
    Size reuse_max_train_{250000};       
    Size reuse_static_files_{1};         
    double reuse_max_shift_{0.5};        
    Size max_train_psms_{500000}; 
    double fdr_psm_{0.0};
    bool fdr_psm_by_scored_charges_{true}; 
    double fdr_protein_{0.0};
    double fdr_protein_inference_psm_{0.1}; 
 
    StringList annotate_psm_;
 
    bool self_trained_ion_priors_{false};       
    FragmentIonLikelihoodModel::ContextSet ion_prior_model_{FragmentIonLikelihoodModel::ContextSet::RICH}; 
    bool ion_prior_scored_peaks_{false};        
    int ion_prior_max_fragment_charge_{2};      
    double ion_prior_train_fdr_{0.01};          
    Size ion_prior_min_psms_{100};              
    bool ion_prior_feature_llr_{true};          
    bool ion_prior_feature_explained_{true};    
    bool ion_prior_feature_topk_{false};        
 
    Size peptide_min_size_;
    Size peptide_max_size_;
    Size peptide_missed_cleavages_;
    EnzymaticDigestion::Specificity peptide_enzyme_specificity_{EnzymaticDigestion::SPEC_FULL};
 
 
    Size report_top_hits_;
    Size rescore_top_hits_{0};
    Size rescoreTopHits_() const
    {
      return rescore_top_hits_ > 0 ? rescore_top_hits_ : report_top_hits_;
    }
    Size searchTopHits_() const
    {
      if (!rescore_ || decoy_mode_ == DecoyMode_::IGNORE) return report_top_hits_;
      return keep_pre_fdr_psms_ ? std::max(report_top_hits_, rescoreTopHits_()) : rescoreTopHits_();
    }
    bool isotope_error_observed_minus_theoretical_{true};
 
    bool add_a_ions_{false};
    bool add_b_ions_{true};
    bool add_c_ions_{false};
    bool add_x_ions_{false};
    bool add_y_ions_{true};
    bool add_z_ions_{false};
    bool add_zp1_ions_{false};
    bool ions_by_activation_{true}; 
    bool protein_mapping_from_index_{true}; 
 
    Size database_chunk_size_{0};  
 
    bool calibration_enabled_{false};
    double calibration_subset_ratio_{0.1};
    bool chimeric_enabled_{false};       
    Size calibration_min_psms_{50};
    double calibration_sigma_factor_{4.0};       
    double calibration_precursor_sigma_factor_{0.0}; 
    double calibration_min_precursor_tol_{0.0};  
    double calibration_min_fragment_tol_{0.0};   
    std::string calibration_recalibrate_{"none"}; 
    Size calibration_recalibrate_rt_chunks_{1};   
 
    static constexpr const char* OBSERVED_PRECURSOR_MZ = "observed_precursor_mz";
    static constexpr const char* OBSERVED_COISOLATED_MZ = "observed_coisolated_mz";
 
    struct OPENMS_DLLAPI MzRecalibration_ // exported: the class test calls its functions (MSVC links nothing of a nested struct without it)
    {
      struct Chunk { double rt; double intercept; double slope; Size points; Size decoys{0}; bool from_neighbours{false}; };
      std::vector<Chunk> chunks; 
      bool ppm{true};            
      bool active() const { return !chunks.empty(); }
      double error(double mz, double rt) const;   
      double correct(double mz, double rt) const; 
      double uncorrect(double corrected, double rt) const; 
      struct OPENMS_DLLAPI AtRT // (exported like its parent: MSVC links nothing of a nested struct without it)
      {
        const Chunk* lo{nullptr}; 
        const Chunk* hi{nullptr}; 
        double w{0.0};
        bool ppm{true};
        double error(double mz) const
        {
          if (lo == nullptr) return 0.0;
          const double e_lo = lo->intercept + lo->slope * mz;
          if (hi == nullptr) return e_lo;
          return (1.0 - w) * e_lo + w * (hi->intercept + hi->slope * mz);
        }
        double correct(double mz) const
        {
          const double e = error(mz);
          return ppm ? mz - e * 1e-6 * mz : mz - e;
        }
      };
      AtRT at(double rt) const; 
      std::string toString() const;
      static MzRecalibration_ fit(const std::vector<double>& errors, const std::vector<double>& mzs,
                                  const std::vector<double>& rts, Size n_chunks, Size min_points, bool linear, bool ppm,
                                  const std::vector<char>* decoys = nullptr);
    };
 
    struct CalibrationResult_
    {
      double precursor_shift{0};     
      double precursor_sigma{0};     
      double precursor_raw_shift{0}; 
      double precursor_raw_sigma{0};
      double fragment_raw_shift{0};  
      double fragment_raw_sigma{0};
      MzRecalibration_ precursor_recalibration; 
      MzRecalibration_ fragment_recalibration;
      bool recalibrated() const { return precursor_recalibration.active() || fragment_recalibration.active(); }
      const MzRecalibration_* appliedFragmentRecalibration() const { return success && fragment_recalibration.active() ? &fragment_recalibration : nullptr; }
      double precursor_spread{0};    
      double cal_lower{0};           
      double cal_upper{0};           
      double fragment_tolerance{0};  
      double fragment_shift{0};      
      double fragment_sigma{0};      
      double precursor_coverage{0};  
      double fragment_coverage{0};   
      Size psms{0};                  
      double precursor_measured_coverage{0};
      double fragment_measured_coverage{0};
      bool precursor_measurement_trusted{false};
      bool fragment_measurement_trusted{false};
      Size fragment_peaks{0};        
      bool extreme_bias{false};      
      bool success{false};           
    };
 
    mutable CalibrationResult_ last_calibration_result_;
 
    mutable RunStatistics last_run_stats_;
 
    mutable PercolatorRescoring last_percolator_;
    bool keep_pre_fdr_psms_ = false; 
    FileResultConsumer file_result_consumer_; 
    std::string parameter_name_prefix_; 
    std::string paramName_(const std::string& name) const { return parameter_name_prefix_ + name; }
    mutable bool pool_rescoring_ = false;
    bool keepPreRescoringPsms_() const { return keep_pre_fdr_psms_ || pool_rescoring_; }
    mutable Size pool_files_ = 0;
    Size transferQuota_() const;
    static std::vector<Size> transferSample_(const PeptideIdentificationList& ids, Size max_psms, Size hits_per_spectrum);
    mutable PeptideIdentificationList pre_fdr_peptide_ids_;
    mutable std::shared_ptr<void> released_hits_;
 
    struct ReuseScoring_;
    struct ReuseModel_;
    struct ReuseRun_
    {
      Size file_index;
      const std::vector<ProteinIdentification>* protein_ids;
      PeptideIdentificationList* peptide_ids;
    };
 
    static PercolatorRescoring rescoreWithPercolator_(std::vector<ProteinIdentification>& protein_ids,
                                                      PeptideIdentificationList& peptide_ids,
                                                      int threads,
                                                      std::vector<std::vector<PeptideHit>>* dropped,
                                                      const PercolatorTraining& training,
                                                      const Param& extra,
                                                      const ReuseScoring_* reuse = nullptr);
 
    std::shared_ptr<const ReuseModel_> trainReuseModel_(const std::vector<ReuseRun_>& runs, int threads) const;
 
    PercolatorRescoring rescoreWithReuseModel_(std::vector<ProteinIdentification>& protein_ids,
                                               PeptideIdentificationList& peptide_ids,
                                               Size file_index,
                                               const ReuseModel_& model,
                                               bool check_shift,
                                               int threads,
                                               std::vector<std::vector<PeptideHit>>* dropped) const;
 
    void applyRescoredPsmFDR_(std::vector<ProteinIdentification>& protein_ids,
                              PeptideIdentificationList& peptide_ids,
                              bool has_decoys,
                              RunStatistics& stats,
                              bool rescored,
                              PeptideIdentificationList* inference_only_psms) const;
 
    void finishDeferredRescoring_(MultiFileSearchResult& mfres, const std::vector<Size>& deferred) const;
 
    mutable std::shared_ptr<const ReuseModel_> reuse_model_;
    mutable bool reuse_defer_ = false;
    mutable Size reuse_file_index_ = 0;
    mutable PeptideIdentificationList inference_only_peptide_ids_;
 
    mutable double last_mod_match_tolerance_used_{-1.0};
 
    double computeModMatchTolerance_() const
    {
      if (precursor_mass_tolerance_lower_ <= 0.0) return precursor_mass_tolerance_upper_;
      if (precursor_mass_tolerance_upper_ <= 0.0) return precursor_mass_tolerance_lower_;
      return std::min(precursor_mass_tolerance_lower_, precursor_mass_tolerance_upper_);
    }
 
    CalibrationResult_ runCalibrationPass_(PeakMap& spectra,
                                           FragmentIndex& fragment_index,
                                           const std::vector<FASTAFile::FASTAEntry>& db,
                                           const PeakMap* query_spectra = nullptr,
                                           const std::string& decoy_string = std::string(),
                                           bool decoy_is_prefix = true) const;
 
    void applyRecalibration_(const CalibrationResult_& cal, PeakMap& spectra, PeakMap* query_spectra,
                             PeakMap* evidence_spectra) const;
 
    void recordRecalibration_(const CalibrationResult_& cal, std::vector<ProteinIdentification>& protein_ids) const;
 
    struct ProteinMapping_
    {
      static constexpr Size SHARDS = 64;
      std::vector<std::string_view> sequences;
      std::vector<std::vector<std::pair<UInt32, UInt32>>> occurrences;
      std::vector<std::unordered_map<std::string_view, Size>> entries;
      std::string fallback_reason;
 
      Size find(std::string_view sequence) const
      {
        if (entries.empty()) return sequences.size();
        const auto& table = entries[std::hash<std::string_view>{}(sequence) % SHARDS];
        const auto entry = table.find(sequence);
        return entry == table.end() ? sequences.size() : entry->second;
      }
    };
 
    static ProteinMapping_ buildProteinMapping_(const FragmentIndex& index,
                                                const std::vector<FASTAFile::FASTAEntry>& db,
                                                std::vector<UInt32> candidates,
                                                const Param& indexer_parameters);
 
    static bool applyProteinMapping_(const ProteinMapping_& mapping,
                                     const std::vector<FASTAFile::FASTAEntry>& db,
                                     const Param& indexer_parameters,
                                     std::vector<ProteinIdentification>& protein_ids,
                                     PeptideIdentificationList& peptide_ids,
                                     std::string& fallback_reason);
 
    static bool accessionHasDecoyMarker_(const std::string& accession,
                                         const std::string& marker, bool is_prefix);
 
    static void removeProteinsReferencedByNeither_(std::vector<ProteinIdentification>& protein_ids,
                                                   const PeptideIdentificationList& peptide_ids,
                                                   const PeptideIdentificationList& also_referencing);
 
    static constexpr const char* SEARCH_ENGINE_SCORE = "ln(hyperscore)";
 
    static bool useSearchEngineScore_(PeptideIdentificationList& ids);
 
    static Size countDecoyEntries_(const std::vector<FASTAFile::FASTAEntry>& db, const std::string& marker, bool is_prefix);
 
    static void poolSearchedFiles_(std::vector<SearchResult>& per_file, bool move_psms, const StringList& in_spectra_files,
                                   std::vector<ProteinIdentification>& protein_ids, PeptideIdentificationList& peptide_ids,
                                   double configured_precursor_tolerance = -1.0, double configured_fragment_tolerance = -1.0);
 
    static bool searched_(const SearchResult& pf);
 
    static int searchedFileOf_(const PeptideIdentification& id, const std::vector<SearchResult>& per_file);
 
    static void returnPooledPsms_(PeptideIdentificationList& pooled, std::vector<SearchResult>& per_file,
                                  const std::vector<std::unordered_map<std::string, double>>* file_peptide_q_values);
 
    static std::vector<std::unordered_map<std::string, double>> filePeptideQValues_(const PeptideIdentificationList& pooled,
                                                                                    Size files);
 
    static void capturePreFdrStats_(const PeptideIdentificationList& peptide_ids,
                                    RunStatistics& stats);
 
    static double maxRetainedScore_(const PeptideIdentificationList& peptide_ids);
 
    static void keepTopHits_(PeptideIdentificationList& peptide_ids, Size n);
 
    PercolatorRescoring applyPsmFDR_(std::vector<ProteinIdentification>& protein_ids,
                                     PeptideIdentificationList& peptide_ids,
                                     bool has_decoys,
                                     RunStatistics& stats,
                                     PeptideIdentificationList* pre_fdr_psms = nullptr,
                                     PeptideIdentificationList* inference_only_psms = nullptr) const;
 
    static void markRescoredByPercolator_(std::vector<ProteinIdentification>& protein_ids, const Param& pp,
                                          const PercolatorTraining& training, const Percolator* perc = nullptr,
                                          const ReuseScoring_* reuse = nullptr);
 
    static bool hasPSMs_(const PeptideIdentificationList& ids);
 
    static void tdcQValues_(PeptideIdentificationList& peptide_ids, bool plus_one = false);
 
 
    void rescoreFilesTogether_(MultiFileSearchResult& mfres, const std::vector<std::string>& in_spectra_files) const;
 
    void collectInferenceOnlyPsms_(const PeptideIdentificationList& peptide_ids, double psm_fdr,
                                   PeptideIdentificationList& inference_only_psms) const;
 
    void collectRunStatistics_(const PeakMap& spectra,
                               const std::vector<ProteinIdentification>& protein_ids,
                               const PeptideIdentificationList& peptide_ids,
                               RunStatistics& stats) const;
 
    struct SpectraCounts_
    {
      Size ms2 = 0;
      Size profile = 0;
      Size charge_out_of_range = 0;
    };
 
    SpectraCounts_ countSpectra_(const PeakMap& spectra) const;
 
    void preprocessSearchedSpectra_(PeakMap& spectra, const std::string& input,
                                    FragmentIonLikelihoodModel::PeakLists& ion_evidence,
                                    PeakMap* evidence_spectra, PeakMap* query_spectra) const;
 
  public:
    static void updateFinalStats(RunStatistics& stats,
                                 const PeptideIdentificationList& peptide_ids,
                                 const std::string& enzyme,
                                 bool fdr_applied);
 
    static void releaseInParallel(SearchResult& result);
 
    static std::string renderTotalTime(const SharedSearchStats& sh);
 
    static void renderRunSummary(const RunStatistics& stats,
                                 const SharedSearchStats& shared,
                                 const OpenSearchModificationAnalysis::OpenSearchAnalysisResult& mod_analysis,
                                 bool is_open_search,
                                 std::ostream& os);
 
    static void renderModificationSummary(const OpenSearchModificationAnalysis::OpenSearchAnalysisResult& mod_analysis,
                                          std::ostream& os);
 
    static std::string renderRunSummaryYaml(
        const MultiFileSearchResult& mfres,
        const std::vector<std::pair<std::string, std::vector<std::string>>>& manifest,
        Size files_failed,
        Size files_total);
 
  private:
 
    bool isOpenSearchMode_() const
    {
      return FragmentIndex::isOpenSearchMode(precursor_mass_tolerance_lower_,
                                             precursor_mass_tolerance_upper_,
                                             precursor_mass_tolerance_unit_ == "ppm");
    }
};
 
} // namespace