Skip to content

LOGMON-240/VDB-5981 treating last part of OP log as freeform, port an… #4

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
aboshkin wants to merge 15 commits into
base: engineering
Choose a base branch
from
Open

LOGMON-240/VDB-5981 treating last part of OP log as freeform, port an… #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
372673f
LOGMON-240/VDB-5981 treating last part of OP log as freeform, port an…
aboshkin May 15, 2025
63e3597
VDB-5981 skipping unrecognized tokens in the free form portion of OP …
aboshkin May 27, 2025
996c868
VDB-5981/LOGMON-240 allow URL query to start and/or end with an &
aboshkin Jun 4, 2025
505f7ce
VDB-5981 added #includes
aboshkin Jun 4, 2025
ae0880c
VDB-5981 added #includes
aboshkin Jun 4, 2025
b6ecd46
VDB-5981/LOGMON-240 allow empty query in URLs
aboshkin Jun 10, 2025
e594b9e
VDB-5981/LOGMON-240 allowed multiple IPs under 'forwarded'
aboshkin Jun 17, 2025
a8c5b37
VDB-5981/LOGMON-240 relaxed the free-form part of the OP grammar
aboshkin Jun 25, 2025
c227ed1
VDB-5981 tolerating more garbage in the request field
aboshkin Jul 10, 2025
a75bbb0
Merge branch 'VDB-5981' of https://github.com/ncbi/ncbi-logging into …
aboshkin Jul 10, 2025
e6a38f7
VDB-5981 made all OP fields optional in the json to OP conversion
aboshkin Jul 10, 2025
e4c16bf
[VDB-5981] Bump versions and set gcc's architecture target
vartanianmh Jul 16, 2025
90825fe
VDB-5981 one more fix in the grammmar; OP json (reverse) parser fixed
aboshkin Jul 18, 2025
f87cbe3
VDB-5981 \n in the request
aboshkin Jul 22, 2025
d5eb07b
VDB-5981 allow escapes inside 'forwarded'
aboshkin Jul 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 16 additions & 16 deletions parser/Makefile.generic
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,22 +57,22 @@ $(OBJDIR)/$(PREFIX)_parser.prof.o: $(OBJDIR)/$(PREFIX)_scanner.prof.o
$(OBJDIR)/$(CPREFIX)_Interface.dbg.o $(OBJDIR)/$(CPREFIX)_Interface.rel.o $(OBJDIR)/$(CPREFIX)_Interface.prof.o : $(PREFIX)_parser.hpp
$(PREFIX)_parser.hpp : $(PREFIX)_parser.cpp

$(FWD_TOOL)-dbg: $($(CPREFIX)2JSN_DBG_OBJ) $(DBG_TOOL_OBJ)
$(LD) -o $@ $^ $(COMMON_DBG_OBJ) $(LDFLAGS)
$(FWD_TOOL)-dbg: $($(CPREFIX)2JSN_DBG_OBJ)
$(LD) -o $@ $^ $(COMMON_DBG_OBJ) $(LDFLAGS_DBG)

$(FWD_TOOL)-rel: $($(CPREFIX)2JSN_REL_OBJ) $(REL_TOOL_OBJ)
$(LD) -o $@ $^ $(COMMON_REL_OBJ) $(LDFLAGS)
$(FWD_TOOL)-rel: $($(CPREFIX)2JSN_REL_OBJ)
$(LD) -o $@ $^ $(COMMON_REL_OBJ) $(LDFLAGS_REL)

$(FWD_TOOL)-prof: $($(CPREFIX)2JSN_PROF_OBJ) $(PROF_TOOL_OBJ)
$(FWD_TOOL)-prof: $($(CPREFIX)2JSN_PROF_OBJ)
$(LD) -o $@ $^ $(COMMON_PROF_OBJ) $(GPROFFLAGS) $(LDFLAGS)

$(REV_TOOL)-dbg: $(JSN2$(CPREFIX)_DBG_OBJ) $(DBG_TOOL_OBJ)
$(LD) -o $@ $^ $(COMMON_DBG_OBJ) $(LDFLAGS)
$(REV_TOOL)-dbg: $(JSN2$(CPREFIX)_DBG_OBJ)
$(LD) -o $@ $^ $(COMMON_DBG_OBJ) $(LDFLAGS_DBG)

$(REV_TOOL)-rel: $(JSN2$(CPREFIX)_REL_OBJ) $(REL_TOOL_OBJ)
$(LD) -o $@ $^ $(COMMON_REL_OBJ) $(LDFLAGS)
$(REV_TOOL)-rel: $(JSN2$(CPREFIX)_REL_OBJ)
$(LD) -o $@ $^ $(COMMON_REL_OBJ) $(LDFLAGS_REL)

$(REV_TOOL)-prof: $(JSN2$(CPREFIX)_PROF_OBJ) $(PROF_TOOL_OBJ)
$(REV_TOOL)-prof: $(JSN2$(CPREFIX)_PROF_OBJ)
$(LD) -o $@ $^ $(COMMON_PROF_OBJ) $(GPROFFLAGS) $(LDFLAGS)

$(PREFIX): mkdir $(FWD_TOOL)-dbg $(FWD_TOOL)-rel $(REV_TOOL)-dbg $(REV_TOOL)-rel
Expand All @@ -98,24 +98,24 @@ $(PREFIX)_TEST_REVERSE_DBG_OBJ = $(addprefix $(OBJDIR)/,$(subst .cpp,.dbg.o,$($(
$(PREFIX)_TEST_REVERSE_REL_OBJ = $(addprefix $(OBJDIR)/,$(subst .cpp,.rel.o,$($(PREFIX)_TEST_REVERSE_SRC)))

$(BINDIR)/$(PREFIX)_test_parse-dbg: $($(PREFIX)_TEST_PARSE_DBG_OBJ)
$(LD) -o $@ $^ $(COMMON_DBG_OBJ) -lgtest $(LDFLAGS)
$(LD) -o $@ $^ $(COMMON_DBG_OBJ) -lgtest $(LDFLAGS_DBG)

$(BINDIR)/$(PREFIX)_test_parse-rel: $($(PREFIX)_TEST_PARSE_REL_OBJ)
$(LD) -o $@ $^ $(COMMON_REL_OBJ) -lgtest $(LDFLAGS)
$(LD) -o $@ $^ $(COMMON_REL_OBJ) -lgtest $(LDFLAGS_REL)

$(BINDIR)/$(PREFIX)_test_flex-dbg: $($(PREFIX)_TEST_FLEX_DBG_OBJ)
$(LD) -o $@ $^ -lgtest -lpthread
$(LD) -o $@ $^ -lgtest -lpthread $(LDFLAGS_DBG)

$(BINDIR)/$(PREFIX)_test_flex-rel: $($(PREFIX)_TEST_FLEX_REL_OBJ)
$(LD) -o $@ $^ -lgtest -lpthread
$(LD) -o $@ $^ -lgtest -lpthread $(LDFLAGS_REL)

#$(BINDIR)/$(PREFIX)_test_reverse-dbg: $(CPREFIX)_Interface.cpp
$(BINDIR)/$(PREFIX)_test_reverse-dbg: $($(PREFIX)_TEST_REVERSE_DBG_OBJ)
$(LD) -o $@ $^ $(COMMON_DBG_OBJ) -lgtest $(LDFLAGS)
$(LD) -o $@ $^ $(COMMON_DBG_OBJ) -lgtest $(LDFLAGS_DBG)

#$(BINDIR)/$(PREFIX)_test_reverse-rel: $(CPREFIX)_Interface.cpp
$(BINDIR)/$(PREFIX)_test_reverse-rel: $($(PREFIX)_TEST_REVERSE_REL_OBJ)
$(LD) -o $@ $^ $(COMMON_REL_OBJ) -lgtest $(LDFLAGS)
$(LD) -o $@ $^ $(COMMON_REL_OBJ) -lgtest $(LDFLAGS_REL)

unittest_parse: $(PREFIX) flex_test $(BINDIR)/$(PREFIX)_test_parse-dbg $(BINDIR)/$(PREFIX)_test_parse-rel
$(BINDIR)/$(PREFIX)_test_parse-dbg
Expand Down
15 changes: 11 additions & 4 deletions parser/Makefile.set
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ $(OBJDIR)/%.dbg.o: %.cpp
$(CPP) $(CXXFLAGS) -g $< -o $(OBJDIR)/$(*F).dbg.o

$(OBJDIR)/%.rel.o: %.cpp
$(CPP) $(CXXFLAGS) -O3 $< -o $(OBJDIR)/$(*F).rel.o
$(CPP) $(CXXFLAGS) -march=skylake -O3 $< -o $(OBJDIR)/$(*F).rel.o
# lmem12=skylake, lmem14=cascade lake

$(OBJDIR)/%.prof.o: %.cpp
$(CPP) $(CXXFLAGS) $(GPROFFLAGS) $< -o $(OBJDIR)/$(*F).prof.o
Expand All @@ -34,11 +35,17 @@ $(OBJDIR)/%.prof.o: %.cpp

#################################################
# Specify location of ncbi-vdb3/jwt-tool in JWT_TOOL if different
JWT_TOOL ?= ../../../ncbi-vdb3/jwt-tool
JWT_TOOL ?= $(abspath ../../../ncbi-vdb3/jwt-tool)

CXXFLAGS += -I$(JWT_TOOL)/inc -I$(JWT_TOOL)/utf8proc -I$(JWT_TOOL)/tool
GTEST ?= $(JWT_TOOL)/../external/googletest
GTESTLIB ?= $(JWT_TOOL)/../external/lib

LDFLAGS = -L$(JWT_TOOL)/lib -lncbi-json-dbg -lncbi-secure-dbg -lutf8proc -lmbedcrypto -lpthread
CXXFLAGS += -I$(JWT_TOOL)/inc -I$(JWT_TOOL)/utf8proc -I$(JWT_TOOL)/tool -I $(GTEST)/googletest/include

LDFLAGS = -L$(JWT_TOOL)/lib -L$(GTESTLIB) -lutf8proc -lmbedcrypto -lpthread

LDFLAGS_DBG = -lncbi-json-dbg -lncbi-secure-dbg $(LDFLAGS)
LDFLAGS_REL = -lncbi-json-rel -lncbi-secure-rel $(LDFLAGS)

#################################################

Expand Down
1 change: 1 addition & 0 deletions parser/aws/AWS_Interface.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#include "ParserInterface.hpp"

#include <memory>
#include <string>

namespace NCBI
{
Expand Down
1 change: 1 addition & 0 deletions parser/cl/CL_Interface.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#include "ParserInterface.hpp"

#include <memory>
#include <string>

namespace NCBI
{
Expand Down
19 changes: 12 additions & 7 deletions parser/common/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# COMMON_SRC
include ../Makefile.set

all: cmd common
all: common

test: cmn_test

Expand All @@ -20,18 +20,23 @@ mkdir:
CMD_SRC_LOCATION = $(JWT_TOOL)/tool
CMD_SRC = $(CMD_SRC_LOCATION)/cmdline.cpp

CMD_DBG_OBJ = $(addprefix $(OBJDIR)/,$(subst .cpp,.dbg.o,$(notdir $(CMD_SRC))))
CMD_REL_OBJ = $(addprefix $(OBJDIR)/,$(subst .cpp,.rel.o,$(notdir $(CMD_SRC))))
CMD_DBG_OBJ = $(addprefix $(OBJDIR)/,$(subst .cpp,.dbg.o,$(notdir $(CMD_SRC))))
CMD_REL_OBJ = $(addprefix $(OBJDIR)/,$(subst .cpp,.rel.o,$(notdir $(CMD_SRC))))
CMD_PROF_OBJ = $(addprefix $(OBJDIR)/,$(subst .cpp,.prof.o,$(notdir $(CMD_SRC))))

$(info CMD_SRC_LOCATION=$(CMD_SRC_LOCATION))
$(info $(OBJDIR)/%.dbg.o: $(CMD_SRC_LOCATION)/%.cpp)

$(OBJDIR)/%.dbg.o: $(CMD_SRC_LOCATION)/%.cpp
$(CPP) $(CXXFLAGS) -g $< -o $(OBJDIR)/$(*F).dbg.o

$(OBJDIR)/%.rel.o: $(CMD_SRC_LOCATION)/%.cpp
$(CPP) $(CXXFLAGS) -O3 $< -o $(OBJDIR)/$(*F).rel.o
$(CPP) $(CXXFLAGS) -O3 -march=skylake $< -o $(OBJDIR)/$(*F).rel.o

$(OBJDIR)/%.prof.o: $(CMD_SRC_LOCATION)/%.cpp
$(CPP) $(CXXFLAGS) -O3 $< -o $(OBJDIR)/$(*F).prof.o
$(CPP) $(CXXFLAGS) -O3 -march=skylake $< -o $(OBJDIR)/$(*F).prof.o

$(info cmd: mkdir $(CMD_DBG_OBJ) $(CMD_REL_OBJ) $(CMD_PROF_OBJ))

cmd: mkdir $(CMD_DBG_OBJ) $(CMD_REL_OBJ) $(CMD_PROF_OBJ)

Expand Down Expand Up @@ -110,10 +115,10 @@ cmn_test: mkdir $(BINDIR)/test_common-dbg $(BINDIR)/test_common-rel
cmn_prof: mkdir $(COMMON_PROF_OBJ) $(CMD_PROF_OBJ)

$(BINDIR)/test_common-dbg: $(TEST_SUPPORT_DBG_OBJ) $(CMD_DBG_OBJ)
$(LD) -o $@ $^ -L$(JWT_TOOL)/lib -lncbi-json-dbg -lncbi-secure-dbg -lutf8proc -lmbedcrypto -lgtest -lpthread
$(LD) -o $@ $^ $(LDFLAGS_DBG) -lgtest

$(BINDIR)/test_common-rel: $(TEST_SUPPORT_REL_OBJ) $(CMD_REL_OBJ)
$(LD) -o $@ $^ -L$(JWT_TOOL)/lib -lncbi-json-rel -lncbi-secure-rel -lutf8proc -lmbedcrypto -lgtest -lpthread
$(LD) -o $@ $^ $(LDFLAGS_DBG) -lgtest

.PHONY: common cmn_clean cmn_test

Expand Down
15 changes: 14 additions & 1 deletion parser/common/ParserInterface.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,24 @@ ParseBlockInterface :: receive_one_line( const char * line, size_t line_size, si

if ( ! format_specific_parse( line, line_size ) )
{
if ( m_debug)
{
cout<<"ParseBlockInterface :: receive_one_line(): format_specific_parse failed " << endl;
}
receiver . SetCategory( ReceiverInterface::cat_ugly );
}
else if ( receiver . GetCategory() == ReceiverInterface::cat_good )
{
receiver . SetCategory( receiver.post_process() );
if ( m_debug)
{
cout<<"ParseBlockInterface :: receive_one_line(): format_specific_parse succeeded " << endl;
}
ReceiverInterface::Category post_cat = receiver.post_process();
if ( m_debug)
{
cout<<"ParseBlockInterface :: receive_one_line(): post_process = " << post_cat << endl;
}
receiver . SetCategory( post_cat );
}

if ( receiver . GetCategory() != ReceiverInterface::cat_good )
Expand Down
2 changes: 2 additions & 0 deletions parser/common/URL_Interface.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,9 @@ URLParseBlock::format_specific_parse( const char * line, size_t line_size )
url__delete_buffer( bs, m_sc );

if ( ret != 0 )
{
m_receiver . SetCategory( ReceiverInterface::cat_ugly );
}
else if ( m_receiver .GetCategory() == ReceiverInterface::cat_unknown )
m_receiver . SetCategory( ReceiverInterface::cat_good );
return ret == 0;
Expand Down
18 changes: 17 additions & 1 deletion parser/common/test_url_parsing.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -439,4 +439,20 @@ TEST_F( URLTestFixture, LOGMON_217 )
ASSERT_EQ( "NC_000001", extract_value( res, "accession" ) );
ASSERT_EQ( "NC_000001", extract_value( res, "filename" ) );
ASSERT_EQ( ".10.1", extract_value( res, "extension" ) );
}
}

TEST_F( URLTestFixture, LOGMON_240_1_query_starts_with_separator )
{ // query starts with '&'
const std::string res = try_to_parse_good( "/entrez/eutils/esearch.fcgi?&retmode=json&version=2.0&db=taxonomy&term=&", true );
ASSERT_EQ( "", extract_value( res, "accession" ) );
ASSERT_EQ( "esearch", extract_value( res, "filename" ) );
ASSERT_EQ( ".fcgi", extract_value( res, "extension" ) );
}

TEST_F( URLTestFixture, LOGMON_240_duplicated_query_separator )
{ // '&&' in the query
const std::string res = try_to_parse_good( "/phpfm/index.php?&&path=&action=upload" );
ASSERT_EQ( "", extract_value( res, "accession" ) );
ASSERT_EQ( "index", extract_value( res, "filename" ) );
ASSERT_EQ( ".php", extract_value( res, "extension" ) );
}
6 changes: 6 additions & 0 deletions parser/common/url_parser.y
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -288,6 +288,10 @@ query_list
{
$$ = $1;
}
| query_list QUERY_SEP
{
$$ = $1;
}
| query_list QUERY_SEP query_entry
{
// if $3 has more elements than $1, use $3
Expand Down Expand Up @@ -323,6 +327,8 @@ query_list

query
: QMARK query_list { $$ = $2; }
| QMARK QUERY_SEP query_list { $$ = $3; }
| QMARK { Init_Node( $$ ); }
| %empty { Init_Node( $$ ); }
;

Expand Down
2 changes: 1 addition & 1 deletion parser/common/url_scanner.l
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ ACCESSION {SRA}|{REFSEQ1}|{REFSEQ2}|{WGS}|"hs37d5"

SLASH (\/|%2F|%2f)
QMARK "?"
QUERY_SEP [&;]
QUERY_SEP [&;]+
EQUAL "="
HASH "#"
PERCENT "%"
Expand Down
1 change: 1 addition & 0 deletions parser/err/ERR_Interface.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#include "ParserInterface.hpp"

#include <memory>
#include <string>

namespace NCBI
{
Expand Down
4 changes: 2 additions & 2 deletions parser/err/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,10 @@ MSG_TEST_FLEX_DBG_OBJ = $(addprefix $(OBJDIR)/,$(subst .cpp,.dbg.o,$(MSG_TEST_FL
MSG_TEST_FLEX_REL_OBJ = $(addprefix $(OBJDIR)/,$(subst .cpp,.rel.o,$(MSG_TEST_FLEX_SRC)))

$(BINDIR)/msg_test_flex-dbg: $(MSG_TEST_FLEX_DBG_OBJ)
$(LD) -o $@ $^ -lgtest -lpthread
$(LD) -o $@ $^ -lgtest $(LDFLAGS_DBG)

$(BINDIR)/msg_test_flex-rel: $(MSG_TEST_FLEX_REL_OBJ)
$(LD) -o $@ $^ -lgtest -lpthread
$(LD) -o $@ $^ -lgtest $(LDFLAGS_REL)

flex_test: msg_flex_test

Expand Down
1 change: 1 addition & 0 deletions parser/gcp/GCP_Interface.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#include "ParserInterface.hpp"

#include <memory>
#include <string>

namespace NCBI
{
Expand Down
2 changes: 1 addition & 1 deletion parser/gcp/gcp2jsn.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

using namespace NCBI::Logging;

std::string tool_version( "1.1.0" );
std::string tool_version( "1.1.3" );

int main ( int argc, char * argv [], const char * envp [] )
{
Expand Down
Loading