Merge pull request pi-hole#546 from pi-hole/release/v4.3

Pi-hole FTL v4.3

.circleci/config.yml

@@ -14,7 +14,7 @@ version: 2
         command: |
           BRANCH=$([ -z "$CIRCLE_TAG" ] && echo "$CIRCLE_BRANCH" || echo "master")
 
-          make CFLAGS="${CFLAGS}" GIT_BRANCH="${BRANCH}" GIT_TAG="${CIRCLE_TAG}"
+          make GIT_BRANCH="${BRANCH}" GIT_TAG="${CIRCLE_TAG}"
           file pihole-FTL
     - run:
         name: "Upload"
@@ -59,7 +59,6 @@ jobs:
     <<: *job_template
     environment:
       BIN_NAME: "pihole-FTL-linux-x86_32"
-      CFLAGS: "-m32"
 
 workflows:
   version: 2

.gitignore

@@ -19,3 +19,7 @@ version*
 /pihole-FTL.conf
 /pihole-FTL.db
 /pihole-FTL.log
+
+# aux files
+aux/manuf.data
+aux/macvendor.db

FTL.h

@@ -36,8 +36,6 @@
 #include <pwd.h>
 // syslog
 #include <syslog.h>
-// SQLite
-#include "sqlite3.h"
 // tolower()
 #include <ctype.h>
 // Unix socket
@@ -82,17 +80,45 @@
 // can be 24 hours + 59 minutes
 #define OVERTIME_SLOTS ((MAXLOGAGE+1)*3600/OVERTIME_INTERVAL)
 
+// Interval for resolving NEW client and upstream server host names [seconds]
+// Default: 60 (once every minute)
+#define RESOLVE_INTERVAL 60
+
+// Interval for re-resolving ALL known host names [seconds]
+// Default: 3600 (once every hour)
+#define RERESOLVE_INTERVAL 3600
+
 // FTLDNS enums
-enum { DATABASE_WRITE_TIMER, EXIT_TIMER, GC_TIMER, LISTS_TIMER, REGEX_TIMER };
+enum { DATABASE_WRITE_TIMER, EXIT_TIMER, GC_TIMER, LISTS_TIMER, REGEX_TIMER, ARP_TIMER, LAST_TIMER };
 enum { QUERIES, FORWARDED, CLIENTS, DOMAINS, OVERTIME, WILDCARD };
 enum { DNSSEC_UNSPECIFIED, DNSSEC_SECURE, DNSSEC_INSECURE, DNSSEC_BOGUS, DNSSEC_ABANDONED, DNSSEC_UNKNOWN };
-enum { QUERY_UNKNOWN, QUERY_GRAVITY, QUERY_FORWARDED, QUERY_CACHE, QUERY_WILDCARD, QUERY_BLACKLIST, QUERY_EXTERNAL_BLOCKED };
+enum { QUERY_UNKNOWN, QUERY_GRAVITY, QUERY_FORWARDED, QUERY_CACHE, QUERY_WILDCARD, QUERY_BLACKLIST, QUERY_EXTERNAL_BLOCKED_IP, QUERY_EXTERNAL_BLOCKED_NULL, QUERY_EXTERNAL_BLOCKED_NXRA };
 enum { TYPE_A = 1, TYPE_AAAA, TYPE_ANY, TYPE_SRV, TYPE_SOA, TYPE_PTR, TYPE_TXT, TYPE_MAX };
-enum { REPLY_UNKNOWN, REPLY_NODATA, REPLY_NXDOMAIN, REPLY_CNAME, REPLY_IP, REPLY_DOMAIN, REPLY_RRNAME };
+enum { REPLY_UNKNOWN, REPLY_NODATA, REPLY_NXDOMAIN, REPLY_CNAME, REPLY_IP, REPLY_DOMAIN, REPLY_RRNAME, REPLY_SERVFAIL, REPLY_REFUSED, REPLY_NOTIMP, REPLY_OTHER };
 enum { PRIVACY_SHOW_ALL = 0, PRIVACY_HIDE_DOMAINS, PRIVACY_HIDE_DOMAINS_CLIENTS, PRIVACY_MAXIMUM, PRIVACY_NOSTATS };
 enum { MODE_IP, MODE_NX, MODE_NULL, MODE_IP_NODATA_AAAA, MODE_NODATA };
 enum { REGEX_UNKNOWN, REGEX_BLOCKED, REGEX_NOTBLOCKED };
 enum { BLOCKING_DISABLED, BLOCKING_ENABLED, BLOCKING_UNKNOWN };
+enum {
+  DEBUG_DATABASE   = (1 << 0),  /* 00000000 00000001 */
+  DEBUG_NETWORKING = (1 << 1),  /* 00000000 00000010 */
+  DEBUG_LOCKS      = (1 << 2),  /* 00000000 00000100 */
+  DEBUG_QUERIES    = (1 << 3),  /* 00000000 00001000 */
+  DEBUG_FLAGS      = (1 << 4),  /* 00000000 00010000 */
+  DEBUG_SHMEM      = (1 << 5),  /* 00000000 00100000 */
+  DEBUG_GC         = (1 << 6),  /* 00000000 01000000 */
+  DEBUG_ARP        = (1 << 7),  /* 00000000 10000000 */
+  DEBUG_REGEX      = (1 << 8),  /* 00000001 00000000 */
+  DEBUG_API        = (1 << 9),  /* 00000010 00000000 */
+  DEBUG_OVERTIME   = (1 << 10), /* 00000100 00000000 */
+  DEBUG_EXTBLOCKED = (1 << 11), /* 00001000 00000000 */
+  DEBUG_CAPS       = (1 << 12), /* 00010000 00000000 */
+};
+
+// Database table "ftl"
+enum { DB_VERSION, DB_LASTTIMESTAMP, DB_FIRSTCOUNTERTIMESTAMP };
+// Database table "counters"
+enum { DB_TOTALQUERIES, DB_BLOCKEDQUERIES };
 
 // Privacy mode constants
 #define HIDDEN_DOMAIN "hidden"
@@ -107,6 +133,7 @@ typedef struct {
 	char* port;
 	char* db;
 	char* socketfile;
+	char* macvendordb;
 } FTLFileNamesStruct;
 
 typedef struct {
@@ -143,67 +170,69 @@ typedef struct {
 } countersStruct;
 
 typedef struct {
-	bool socket_listenlocal;
-	bool analyze_AAAA;
 	int maxDBdays;
-	bool resolveIPv6;
-	bool resolveIPv4;
 	int DBinterval;
 	int port;
 	int maxlogage;
+	int16_t debug;
 	unsigned char privacylevel;
-	bool ignore_localhost;
 	unsigned char blockingmode;
-	bool regex_debugmode;
+	bool socket_listenlocal;
+	bool analyze_AAAA;
+	bool resolveIPv6;
+	bool resolveIPv4;
+	bool ignore_localhost;
 	bool analyze_only_A_AAAA;
 	bool DBimport;
+	bool parse_arp_cache;
 } ConfigStruct;
 
 // Dynamic structs
 typedef struct {
 	unsigned char magic;
-	time_t timestamp;
-	unsigned int timeidx;
 	unsigned char type;
 	unsigned char status;
+	unsigned char privacylevel;
+	unsigned char reply;
+	unsigned char dnssec;
+	time_t timestamp;
 	int domainID;
 	int clientID;
 	int forwardID;
-	sqlite3_int64 db;
 	int id; // the ID is a (signed) int in dnsmasq, so no need for a long int here
-	bool complete;
-	unsigned char privacylevel;
 	unsigned long response; // saved in units of 1/10 milliseconds (1 = 0.1ms, 2 = 0.2ms, 2500 = 250.0ms, etc.)
-	unsigned char reply;
-	unsigned char dnssec;
-	bool AD;
+	int64_t db;
+	unsigned int timeidx;
+	bool complete;
 } queriesDataStruct;
 
 typedef struct {
 	unsigned char magic;
+	size_t ippos;
+	size_t namepos;
 	int count;
 	int failed;
-	unsigned long long ippos;
-	unsigned long long namepos;
 	bool new;
 } forwardedDataStruct;
 
 typedef struct {
 	unsigned char magic;
+	size_t ippos;
+	size_t namepos;
+	time_t lastQuery;
 	int count;
 	int blockedcount;
-	unsigned long long ippos;
-	unsigned long long namepos;
-	bool new;
 	int overTime[OVERTIME_SLOTS];
+	unsigned int numQueriesARP;
+	bool new;
 } clientsDataStruct;
 
 typedef struct {
 	unsigned char magic;
+	unsigned char regexmatch;
+	size_t domainpos;
 	int count;
 	int blockedcount;
-	unsigned long long domainpos;
-	unsigned char regexmatch;
 } domainsDataStruct;
 
 typedef struct {
@@ -217,8 +246,8 @@ typedef struct {
 } overTimeDataStruct;
 
 typedef struct {
-	int count;
 	char **domains;
+	int count;
 } whitelistStruct;
 
 typedef struct {
@@ -228,11 +257,15 @@ typedef struct {
 } ShmSettings;
 
 // Prepare timers, used mainly for debugging purposes
-#define NUMTIMERS 5
+#define NUMTIMERS LAST_TIMER
 
 // Used to check memory integrity in various structs
 #define MAGICBYTE 0x57
 
+// Some magic database constants
+#define DB_FAILED -2
+#define DB_NODATA -1
+
 extern logFileNamesStruct files;
 extern FTLFileNamesStruct FTLfiles;
 extern countersStruct *counters;
@@ -244,34 +277,23 @@ extern clientsDataStruct *clients;
 extern domainsDataStruct *domains;
 extern overTimeDataStruct *overTime;
 
-/// Indexed by client ID, then time index (like `overTime`).
-/// This gets automatically updated whenever a new client or overTime slot is added.
-extern int **overTimeClientData;
-
-extern FILE *logfile;
+// Used in gc.c, memory.c, resolve.c, signals.c, and socket.c
 extern volatile sig_atomic_t killed;
-
-extern char ** setupVarsArray;
-extern int setupVarsElements;
-
-extern bool initialscan;
-extern bool debug;
-extern bool threadwritelock;
-extern bool threadreadlock;
+// Used in api.c, grep.c, and dnsmasq_interface.c
 extern unsigned char blockingstatus;
-
+// Used in main.c, log.c, and others
 extern char * username;
-extern char timestamp[16];
-extern bool flush;
-extern bool needGC;
+// Used in main.c, args.c, log.c, and others
 extern bool daemonmode;
+// Used in main.c, database.c, and others
 extern bool database;
+// Used in database.c and gc.c
 extern long int lastdbindex;
-extern bool travis;
+// Used in database.c and gc.c
 extern bool DBdeleteoldqueries;
-extern bool rereadgravity;
-extern long int lastDBimportedtimestamp;
+// Used in main.c, socket.c, and dnsmasq_interface.c
 extern bool ipv4telnet, ipv6telnet;
+// Used in api.c, and socket.c
 extern bool istelnet[MAXCONNS];
 
 // Use out own memory handling functions that will detect possible errors
@@ -286,7 +308,7 @@ extern bool istelnet[MAXCONNS];
 #define realloc(p1,p2) FTLrealloc(p1,p2, __FILE__,  __FUNCTION__,  __LINE__)
 
 extern int argc_dnsmasq;
-extern char **argv_dnsmasq;
+extern const char ** argv_dnsmasq;
 
 extern pthread_t telnet_listenthreadv4;
 extern pthread_t telnet_listenthreadv6;

Makefile

@@ -14,9 +14,9 @@ DNSMASQOPTS = -DHAVE_DNSSEC -DHAVE_DNSSEC_STATIC
 # Flags for compiling with libidn2: -DHAVE_LIBIDN2 -DIDN2_VERSION_NUMBER=0x02000003
 
 FTLDEPS = FTL.h routines.h version.h api.h dnsmasq_interface.h shmem.h
-FTLOBJ = main.o memory.o log.o daemon.o datastructure.o signals.o socket.o request.o grep.o setupVars.o args.o gc.o config.o database.o msgpack.o api.o dnsmasq_interface.o resolve.o regex.o shmem.o overTime.o
+FTLOBJ = main.o memory.o log.o daemon.o datastructure.o signals.o socket.o request.o grep.o setupVars.o args.o gc.o config.o database.o msgpack.o api.o dnsmasq_interface.o resolve.o regex.o shmem.o capabilities.o networktable.o overTime.o
 
-DNSMASQDEPS = config.h dhcp-protocol.h dns-protocol.h radv-protocol.h dhcp6-protocol.h dnsmasq.h ip6addr.h metrics.h
+DNSMASQDEPS = config.h dhcp-protocol.h dns-protocol.h radv-protocol.h dhcp6-protocol.h dnsmasq.h ip6addr.h metrics.h ../dnsmasq_interface.h
 DNSMASQOBJ = arp.o dbus.o domain.o lease.o outpacket.o rrfilter.o auth.o dhcp6.o edns0.o log.o poll.o slaac.o blockdata.o dhcp.o forward.o loop.o radv.o tables.o bpf.o dhcp-common.o helper.o netlink.o rfc1035.o tftp.o cache.o dnsmasq.o inotify.o network.o rfc2131.o util.o conntrack.o dnssec.o ipset.o option.o rfc3315.o crypto.o dump.o ubus.o metrics.o
 
 # Get git commit version and date
@@ -26,8 +26,12 @@ GIT_VERSION := $(shell git --no-pager describe --tags --always --dirty)
 GIT_DATE := $(shell git --no-pager show --date=short --format="%ai" --name-only | head -n 1)
 GIT_TAG := $(shell git describe --tags --abbrev=0)
 
-# -fstack-protector: The program will be resistant to having its stack overflowed
-# -D_FORTIFY_SOURCE=2 and -O1 or higher: This causes certain unsafe glibc functions to be replaced with their safer counterparts
+# Is compiler at least gcc version 8? We cannot do ifgt in Makefile, so we use the shell expr command
+GCCVERSION8 := $(shell expr `$(CC) -dumpversion | cut -f1 -d.` \>= 8)
+
+# Code hardening and debugging improvements
+# -fstack-protector-strong: The program will be resistant to having its stack overflowed
+# -Wp,-D_FORTIFY_SOURCE=2 and -O1 or higher: This causes certain unsafe glibc functions to be replaced with their safer counterparts
 # -Wl,-z,relro: reduces the possible areas of memory in a program that can be used by an attacker that performs a successful memory corruption exploit
 # -Wl,-z,now: When combined with RELRO above, this further reduces the regions of memory available to memory corruption attacks
 # -pie -fPIE: For ASLR (address space layout randomization)
@@ -38,17 +42,20 @@ GIT_TAG := $(shell git describe --tags --abbrev=0)
 # -Wl,-z,defs: Detect and reject underlinking (phenomenon caused by missing shared library arguments when invoking the linked editor to produce another shared library)
 # -Wl,-z,now: Disable lazy binding
 # -Wl,-z,relro: Read-only segments after relocation
-HARDENING_FLAGS=-fstack-protector -D_FORTIFY_SOURCE=2 -O3 -Wl,-z,relro,-z,now -pie -fPIE -fexceptions -fasynchronous-unwind-tables -Wl,-z,defs -Wl,-z,now -Wl,-z,relro
+HARDENING_FLAGS=-fstack-protector-strong -Wp,-D_FORTIFY_SOURCE=2 -O3 -Wl,-z,relro,-z,now -pie -fPIE -fexceptions -fasynchronous-unwind-tables -Wl,-z,defs -Wl,-z,now -Wl,-z,relro
 DEBUG_FLAGS=-rdynamic -fno-omit-frame-pointer
+
 # -DSQLITE_OMIT_LOAD_EXTENSION: This option omits the entire extension loading mechanism from SQLite, including sqlite3_enable_load_extension() and sqlite3_load_extension() interfaces. (needs -ldl linking option, otherwise)
 # -DSQLITE_DEFAULT_MEMSTATUS=0: This setting causes the sqlite3_status() interfaces that track memory usage to be disabled. This helps the sqlite3_malloc() routines run much faster, and since SQLite uses sqlite3_malloc() internally, this helps to make the entire library faster.
 # -DSQLITE_OMIT_DEPRECATED: Omitting deprecated interfaces and features will not help SQLite to run any faster. It will reduce the library footprint, however. And it is the right thing to do.
 # -DSQLITE_OMIT_PROGRESS_CALLBACK: The progress handler callback counter must be checked in the inner loop of the bytecode engine. By omitting this interface, a single conditional is removed from the inner loop of the bytecode engine, helping SQL statements to run slightly faster.
 SQLITEFLAGS=-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_MEMORYDB
+
 # -Wall: This enables all the warnings about constructions that some users consider questionable, and that are easy to avoid (or modify to prevent the warning), even in conjunction with macros. This also enables some language-specific warnings described in C++ Dialect Options and Objective-C and Objective-C++ Dialect Options.
 # -Wextra: This enables some extra warning flags that are not enabled by -Wall.
 # -Wno-unused-parameter: Disable warning for unused parameters. For threads that don't need arguments, we still have to provide a void* args which is then unused.
 WARNFLAGS=-Wall -Wextra -Wno-unused-parameter
+
 # Extra warning flags we apply only to the FTL part of the code (used not for foreign code such as dnsmasq and SQLite3)
 # -Werror: Halt on any warnings, useful for enforcing clean code without any warnings (we use it only for our code part)
 # -Waddress: Warn about suspicious uses of memory addresses
@@ -59,14 +66,39 @@ WARNFLAGS=-Wall -Wextra -Wno-unused-parameter
 # -Wformat-nonliteral: If -Wformat is specified, also warn if the format string is not a string literal and so cannot be checked, unless the format function takes its format arguments as a va_list.
 # -Wuninitialized: Warn if an automatic variable is used without first being initialized
 # -Wswitch-enum: Warn whenever a switch statement has an index of enumerated type and lacks a case for one or more of the named codes of that enumeration.
-EXTRAWARN=-Werror -Waddress -Wlogical-op -Wmissing-field-initializers -Woverlength-strings -Wformat -Wformat-nonliteral -Wuninitialized -Wswitch-enum
+# -Wshadow: Warn whenever a local variable or type declaration shadows another variable, parameter, type, class member, or whenever a built-in function is shadowed.
+# -Wfloat-equal: Warn if floating-point values are used in equality comparisons
+# -Wunsafe-loop-optimizations -funsafe-loop-optimizations: Warn if the loop cannot be optimized because the compiler cannot assume anything on the bounds of the loop indices
+# -Wpointer-arith: Warn about anything that depends on the "size of" a function type or of "void".  GNU C assigns these types a size of 1
+# -Wundef: Warn if an undefined identifier is evaluated in an "#if" directive
+# -Wbad-function-cast: Warn when a function call is cast to a non-matching type
+# -Wwrite-strings: When compiling C, give string constants the type "const char[length]" so that copying the address of one into a non-"const" "char *" pointer produces a warning
+# -Wparentheses: Warn if parentheses are omitted in certain contexts, such as when there is an assignment in a context where a truth value is expected, or when operators are nested whose precedence people often get confused about
+# -Wlogical-op: Warn about suspicious uses of logical operators in expressions
+# -Wstrict-prototypes: Warn if a function is declared or defined without specifying the argument types
+# -Wmissing-prototypes: Warn if a global function is defined without a previous prototype declaration
+# -Wredundant-decls: Warn if anything is declared more than once in the same scope
+# -Winline: Warn if a function that is declared as inline cannot be inlined
+ifeq "$(GCCVERSION8)" "1"
+  # -Wduplicated-cond: Warn about duplicated conditions in an if-else-if chain
+  # -Wduplicated-branches: Warn when an if-else has identical branches
+  # -Wcast-align=strict: Warn whenever a pointer is cast such that the required alignment of the target is increased. For example, warn if a "char *" is cast to an "int *" regardless of the target machine.
+  # -Wlogical-not-parentheses: Warn about logical not used on the left hand side operand of a comparison
+  EXTRAWARNGCC8=-Wduplicated-cond -Wduplicated-branches -Wcast-align=strict -Wlogical-not-parentheses -Wsuggest-attribute=pure -Wsuggest-attribute=const -Wsuggest-attribute=noreturn -Wsuggest-attribute=malloc -Wsuggest-attribute=format -Wsuggest-attribute=cold
+else
+  EXTRAWARNGCC8=
+endif
+EXTRAWARN=-Werror -Waddress -Wlogical-op -Wmissing-field-initializers -Woverlength-strings -Wformat -Wformat-nonliteral -Wuninitialized -Wswitch-enum -Wshadow \
+-Wfloat-equal -Wunsafe-loop-optimizations -funsafe-loop-optimizations -Wbad-function-cast -Wwrite-strings -Wparentheses -Wlogical-op -Wstrict-prototypes -Wmissing-prototypes -Wredundant-decls -Winline $(EXTRAWARNGCC8)
+
 # -FILE_OFFSET_BITS=64: used by stat(). Avoids problems with files > 2 GB on 32bit machines
 CCFLAGS=-std=gnu11 -I$(IDIR) $(WARNFLAGS) -D_FILE_OFFSET_BITS=64 $(HARDENING_FLAGS) $(DEBUG_FLAGS) $(CFLAGS) $(SQLITEFLAGS)
+
 # for FTL we need the pthread library
 # for dnsmasq we need the nettle crypto library and the gmp maths library
 # We link the two libraries statically. Although this increases the binary file size by about 1 MB, it saves about 5 MB of shared libraries and makes deployment easier
 #LIBS=-pthread -lnettle -lgmp -lhogweed
-LIBS=-pthread -Wl,-Bstatic -L/usr/local/lib -lhogweed -lgmp -lnettle  -Wl,-Bdynamic -lrt
+LIBS=-pthread -lrt -Wl,-Bstatic -L/usr/local/lib -lhogweed -lgmp -lnettle -Wl,-Bdynamic
 # Flags for compiling with libidn : -lidn
 # Flags for compiling with libidn2: -lidn2