More graceful handling of full documents errors

[bio-boris]

This spams the logs and makes the logs less useful.

[MrCreosote]

Is that from adding logs to a job? if so, maybe the logs should be split out into 1 doc per log entry

[bio-boris]

Maybe. but why are apps making so many logs in the first place? do they really need to? If we do want so many log lines, why can users only see 10k logs in the job browser? Whats the point of keeping them if the user cannot even see them (unless they are calling the job logs api) and the causes the browser to crash to attempt to load in so many log lines.

Not exactly related, but in addition, I think that logs should expire after some time as well like they do on github actions.

[MrCreosote]

If we're talking about completely redoing logs, then yeah, I would drastically change stuff. Most of the app logs are not fit for human consumption (especially not user consumption)

• Separate logs intended for kbase admins vs. those intended for users
• Add a mechanism for a job to report progress outside of the logging system that could be displayed in a job list
• Go through the most egregious apps and remove log lines or redirect 3rd party output to a file, which could then be uploaded somewhere if really important

[bio-boris]

Counter({'RAST_SDK.annotate_genomes': 2888,
         'ProkkaAnnotation.annotate_metagenome': 1798,
         'RAST_SDK.rast_genome_assembly': 1110,
         'kb_rnaseq_donwloader.export_rna_seq_expression_as_zip': 1043,
         'kb_BatchApp.run_batch': 1010,
         'fba_tools.build_metagenome_metabolic_model': 969,
         'fba_tools.build_multiple_metabolic_models': 824,
         'kb_uploadmethods.batch_import_assemblies_from_staging': 745,
         'kb_uploadmethods.import_file_as_fba_model_from_staging': 633,
         'RAST_SDK.annotate_genome': 479,
         'kb_Bowtie2.align_reads_to_assembly_app': 477,
         'kb_uploadmethods.import_genbank_from_staging': 436,
         'fba_tools.build_metabolic_model': 409,
         'kb_uploadmethods.unpack_web_file': 318,
         'ProkkaAnnotation.annotate': 274,
         'kb_hmmer.HMMER_dbCAN_Search': 244,
         'kb_hisat2.run_hisat2': 233,
         'FamaProfiling.run_FamaReadProfiling': 232,
         'kb_stringtie.run_stringtie_app': 227,
         'FastANI.fast_ani': 213,
         'kb_hmmer.HMMER_EnvBioelement_Search': 213,
         'ReferenceDataManager.update_loaded_genomes': 205,
         'vConTACT.run_vcontact': 177,
         'kb_cufflinks.run_cufflinks': 160,
         'AssemblyRAST.run_kiki': 146,
         'metabat.run_metabat': 144,
         'kb_StrainFinder.run_StrainFinder_v1': 142,
         'ReferenceDataManager.index_genomes_in_solr': 135,
         'kb_unicycler.run_unicycler': 127,
         'kb_cufflinks.run_Cuffdiff': 125,
         'kb_uploadmethods.upload_fasta_gff_file': 111,
         'AssemblyRAST.run_ray': 107,
         'RAST_SDK.rast_genomes_assemblies': 104,
         'kb_uploadmethods.unpack_staging_file': 94,
         'kb_blast.BLASTn_Search': 93,
         'FamaProfiling.run_FamaGenomeProfiling': 86,
         'hipmer.run_hipmer_hpc': 83,
         'kb_hmmer.HMMER_PhyloMarkers_Search': 81,
         'binqc.bin_qc': 74,
         'fba_tools.bulk_export_objects': 74,
         'kb_virmatcher.run_kb_virmatcher': 72,
         'kb_SetUtilities.KButil_Logical_Slice_Two_FeatureSets': 68,
         'ReferenceDataManager.load_refgenomes': 63,
         'SBMLTools.integrate_model': 63,
         'KBaseRNASeq.CuffdiffCall': 57,
         'AssemblyRAST.run_a6': 54,
         'kb_cocacola.run_kb_cocacola': 53,
         'kb_uploadmethods.batch_import_genomes_from_staging': 50,
         'kb_pickaxe.runpickaxe': 47,
         'PangenomeOrthomcl.build_pangenome_with_orthomcl': 47,
         'fba_tools.excel_file_to_model': 40,
         'Report_util_landml.assembly_metadata_report': 40,
         'kb_meta_decoder.map_reads_to_reference': 39,
         'kb_Msuite.run_checkM_lineage_wf': 37,
         'kb_blast.BLASTx_Search': 36,
         'GenomeFileUtil.genbank_to_genome': 35,
         'kb_gtdbtk.run_kb_gtdbtk': 35,
         'fba_tools.sbml_file_to_model': 34,
         'GenomeImporter.import_external_genome': 34,
         'kb_blast.BLASTp_Search': 32,
         'kb_gtdbtk.run_kb_gtdbtk_classify_wf': 31,
         'kb_concoct.run_kb_concoct': 29,
         'AssemblyRAST.run_arast': 28,
         'ThermoStoichWizard.run_ThermoStoichWizard': 27,
         'ProkkaAnnotation.annotate_contigs': 26,
         'KBaseRNASeq.CufflinksCall': 25,
         'kb_eggnog_mapper.run_eggnog_mapper': 25,
         'Velvet.run_velvet': 24,
         'kb_genomeclassification.upload_trainingset': 24,
         'kb_model_analysis.model_heatmap_analysis': 23,
         'GenericsAPI.collapse_matrix': 23,
         'mightymorphingmodels.morph_model': 21,
         'kb_SPAdes.run_SPAdes': 21,
         'kb_phylogenomics.run_DomainAnnotation_Sets': 21,
         'SBMLTools.sbml_importer': 20,
         'kb_trimmomatic.runTrimmomatic': 20,
         'KBaseRNASeq.Hisat2Call': 19,
         'kb_DRAM.run_kb_dram_annotate': 19,
         'kb_ballgown.run_ballgown_app': 18,
         'kb_dRep.run_dereplicate': 17,
         'kb_DRAM.run_kb_dram_annotate_genome': 17,
         'kb_phylogenomics.build_microbial_speciestree': 16,
         'kb_meta_decoder.call_variants': 16,
         'kb_muscle.MUSCLE_prot': 15,
         'kb_kaiju.run_kaiju': 14,
         'fba_tools.tsv_file_to_model': 13,
         'STAR.run_star': 13,
         'kb_maxbin.run_max_bin': 13,
         'MetagenomeUtils.extract_binned_contigs_as_assembly': 12,
         'kb_DRAM.run_kb_dramv_annotate': 11,
         'kb_deseq.run_deseq2_app': 10,
         'kb_SetUtilities.KButil_Summarize_GenomeSet': 10,
         'kb_genomeclassification.build_classifier': 9,
         'SpeciesTreeBuilder.construct_species_tree': 9,
         'kb_uploadmethods.upload_fastq_file': 8,
         'kb_hmmer.HMMER_MT_Bioelement_Search': 8,
         'kb_phylogenomics.view_fxn_profile': 7,
         'MEGAHIT.run_megahit': 6,
         'metabat.run_metabat_bigmem': 6,
         'kb_staging_exporter.export_to_staging': 6,
         'kb_AssemblyUtilities.run_assembly_depth_of_coverage_with_bbmap': 6,
         'kb_muscle.MUSCLE_nuc': 5,
         'simpleapp.simple_add_multiprocessing': 5,
         'GenericsAPI.import_matrix_from_biom': 5,
         'kb_assembly_compare.run_contig_distribution_compare': 5,
         'kb_uploadmethods.import_fasta_as_assembly_from_staging': 5,
         'RAST_SDK.annotate_metagenome': 5,
         'kb_reaction_gene_finder.find_genes_from_similar_reactions': 5,
         'kb_genomeclassification.rast_annotate_trainingset': 5,
         'kb_model_analysis.model_set_to_functional_profiles': 5,
         'kb_uploadmethods.import_tsv_or_excel_as_media_from_staging': 5,
         'kb_cutadapt.remove_adapters': 5,
         'MergeMetabolicAnnotations.import_bulk_annotations': 5,
         'jgi_mg_assembly.run_mg_assembly_pipeline': 4,
         'mags_mash.run_mags_mash': 4,
         'Check16S.check_16S': 4,
         'CompoundSetUtils.compound_set_from_file': 4,
         'fba_tools.compare_models': 3,
         'genome_transform.genbank_to_genome': 3,
         'fba_tools.simulate_growth_on_phenotype_data': 3,
         'kb_tophat2.run_tophat2_app': 3,
         'kb_SPAdes.run_metaSPAdes': 3,
         'kb_genomeclassification.predict_phenotype': 3,
         'kb_hmmer.HMMER_Local_MSA_Group_Search': 3,
         'kb_ReadsUtilities.KButil_Split_Reads': 3,
         'KBAnnotationApps.PDBAnnotation': 3,
         'kb_ObjectInfo.assembly_metadata_report': 3,
         'GLM4EC.annotate_microbes_with_GLM4EC': 3,
         'kb_util_dylan.KButil_Extract_Unpaired_Reads_and_Synchronize_Pairs': 2,
         'kb_phylogenomics.view_pan_circle_plot': 2,
         'kb_Metrics.count_ncbi_genome_features': 2,
         'reference_data_update.fix_genome_taxonomies': 2,
         'fba_tools.compare_flux_with_expression': 2,
         'kb_phylogenomics.find_homologs_with_genome_context': 2,
         'fba_tools.predict_auxotrophy': 2,
         'kb_hmmer.HMMER_MSA_Search': 2,
         'VirSorter.run_VirSorter': 2,
         'kb_SetUtilities.KButil_Batch_Create_AssemblySet': 2,
         'dRep.dereplicate': 2,
         'kb_das_tool.run_kb_das_tool': 2,
         'ModelSEEDReconstruction.build_metabolic_models': 2,
         'kb_SetUtilities.KButil_Batch_Create_GenomeSet': 2,
         'WholeGenomeAlignment.run_mugsy': 1,
         'WholeGenomeAlignment.run_mauve': 1,
         'AssemblyRAST.run_velvet': 1,
         'kb_ea_utils.run_Fastq_Multx': 1,
         'genome_transform.genbank_to_genome_GFU': 1,
         'kb_Msuite.run_checkM_lineage_wf_withFilter': 1,
         'kb_meta_decoder.calculate_population_statistics': 1,
         'kb_cmash.run_kb_cmash': 1,
         'cello.run_cello': 1,
         'GenomeComparisonSDK.build_pangenome': 1,
         'kb_SPAdes.run_HybridSPAdes': 1,
         'MetaAnnot.annotate_contigs': 1,
         'FamaProfiling.run_FamaProteinProfiling': 1,
         'kb_ObjectUtilities.KButil_Concat_MSAs': 1,
         'kb_meta_decoder.call_variants_single': 1,
         'kb_assembly_compare.run_filter_contigs_by_length': 1,
         'rbts_genome_to_genetable.run_rbts_genome_to_genetable': 1,
         'kb_SetUtilities.KButil_Build_GenomeSet': 1,
         'GenomeFileUtil.export_genome_as_genbank': 1,
         'omreegalozpathway_completeness.run_omreegalozpathway_completeness': 1,
         'FeatureSetUtils.upload_featureset_from_diff_expr': 1})