Add support for Commons files

Rewritten in C++ because the C++ library BBHash helps avoid loading
entire keyfile into RAM.

Author: Ordoviz

.gitignore

@@ -1,6 +1,2 @@
-commonscats-in-commons.txt
-commonscats-in-osm.xml
-*.gz
-*.geojson
-*.tsv
-*.pbf
+data/*
+out/*

LICENSE

@@ -1,4 +1,5 @@
-Copyright (c) 2023 Lennard Hofmann
+Copyright (c) 2015 Guillaume Rizk (BBHash)
+Copyright (c) 2023 Lennard Hofmann (CommonsChecker4OSM)
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -1,34 +1,53 @@
-.PHONY: geojson
-geojson: out.geojson out-maproulette.geojson
+yesterday != date -d yesterday +"%Y%m%d"
 
-.PHONY: tsv
-tsv: out.tsv
+CXX ?= g++
+CFLAGS = -O3 -std=c++17 -lpthread
 
-planet.pbf:
-	@echo "Missing planet.pbf: See https://wiki.openstreetmap.org/wiki/Planet.osm#Downloading for how to download an Osmium-compatible planet file." >&2; exit 1
-	# You could add a command here to download planet.pbf with a BitTorrent client
+main: src/main.cpp src/BooPHF.h
+	$(CXX) -o $@ $< $(CFLAGS)
 
-commonswiki-all-titles.gz: FORCE
-	curl -z commonswiki-all-titles.gz https://dumps.wikimedia.org/commonswiki/latest/commonswiki-latest-all-titles.gz -o commonswiki-all-titles.gz
 
-commonscats-in-commons.txt: commonswiki-all-titles.gz
+### NECESSARY INTERMEDIATE FILES: --------------------------------
+
+data/planet.pbf:
+	@echo "Missing data/planet.pbf: See https://wiki.openstreetmap.org/wiki/Planet.osm#Downloading for how to download an Osmium-compatible planet file." >&2; exit 1
+	# You could add a command here to download data/planet.pbf with a BitTorrent client
+
+data/commonswiki-all-titles.gz: FORCE
+	curl -z data/commonswiki-all-titles.gz https://dumps.wikimedia.org/commonswiki/latest/commonswiki-latest-all-titles.gz -o data/commonswiki-all-titles.gz
+
+data/commonswiki-cats.txt: data/commonswiki-all-titles.gz
 	# namespace 14 is "Category:"
-	zcat commonswiki-all-titles.gz | grep ^14 | cut -f2  > commonscats-in-commons.txt
+	zcat data/commonswiki-all-titles.gz | grep ^14 | cut -f2  > data/commonswiki-cats.txt
+
+data/commonswiki-files.txt.gz: FORCE
+	curl -z data/commonswiki-files.txt.gz https://dumps.wikimedia.org/other/mediatitles/$(yesterday)/commonswiki-$(yesterday)-all-media-titles.gz -o data/commonswiki-files.txt.gz
+
+data/commonswiki-files.txt: data/commonswiki-files.txt.gz
+	gunzip --keep data/commonswiki-files.txt.gz
+
+data/planet-filtered.tsv: data/planet.pbf
+	osmium tags-filter -R data/planet.pbf 'nwr/wikimedia_commons' -f xml | ./xml2tsv.py > data/planet-filtered.tsv
+
+data/planet-filtered.geojson: data/planet.pbf
+	osmium tags-filter -t data/planet.pbf 'nwr/wikimedia_commons' -o data/planet-filtered.pbf --overwrite
+	osmium export data/planet-filtered.pbf -c src/config.json -o data/planet-filtered.geojson -f jsonseq --overwrite
+
 
-commonscats-in-osm.xml: planet.pbf
-	osmium tags-filter -R planet.pbf 'nwr/wikimedia_commons=Category:*' -o commonscats-in-osm.xml --overwrite
+### OUTPUT FILES: ------------------------------------------------
 
-planet-filtered.geojson: planet.pbf
-	osmium tags-filter -t planet.pbf 'nwr/wikimedia_commons=Category:*' -o planet-filtered.pbf --overwrite
-	osmium export planet-filtered.pbf -c config.json -o planet-filtered.geojson -f jsonseq --overwrite
+out/cats.tsv: main data/commonswiki-cats.txt data/planet-filtered.tsv
+	# ./main.py Category: data/commonswiki-cats.txt data/planet-filtered.tsv > out/cats.tsv
+	./main Category: $$(wc -l data/commonswiki-cats.txt) data/planet-filtered.tsv $$(nproc) > out/cats.tsv
 
-out.tsv: commonscats-in-osm.xml commonscats-in-commons.txt
-	./main.py out.tsv
+out/cats.geojson: main data/commonswiki-cats.txt data/planet-filtered.geojson
+	# ./main.py Category: data/commonswiki-cats.txt data/planet-filtered.geojson > out/cats.geojson
+	./main Category: $$(wc -l data/commonswiki-cats.txt) data/planet-filtered.geojson $$(nproc) > out/cats.geojson
 
-out.geojson: planet-filtered.geojson commonscats-in-commons.txt
-	./main.py out.geojson
+out/files.tsv: main data/commonswiki-files.txt data/planet-filtered.tsv
+	./main File: $$(wc -l data/commonswiki-files.txt) data/planet-filtered.tsv $$(nproc) > out/files.tsv
 
-out-maproulette.geojson: out.geojson
-	./to_maproulette.py out.geojson > out-maproulette.geojson
+out/files.geojson: main data/commonswiki-files.txt data/planet-filtered.geojson
+	./main File: $$(wc -l data/commonswiki-files.txt) data/planet-filtered.geojson $$(nproc) > out/files.geojson
 
 FORCE: ;

README.md

@@ -1,53 +1,78 @@
-# CommonsCatOSM
+# CommonsChecker4OSM
 
-Finds OpenStreetMap elements tagged with `wikimedia_commons=Category:*` where the category does not exist on Wikimedia Commons.
+Finds OpenStreetMap elements tagged with `wikimedia_commons=Category:*` (or `wikimedia_commons=File:*`) where the category (or file) does not exist on Wikimedia Commons.
+
+## How does it work
+
+CommonsChecker4OSM filters a list of OSM elements tagged wtih `wikimedia_commons=*` against a list of valid Commons categories or files. It is basically an `fgrep -f` implementation that processes a 5 gigabyte pattern file efficiently using the [BBHash](https://github.com/rizkg/BBHash) minimal perfect hashing library. A Makefile is provided to download and prepare the input files. A Python script [main.py](./main.py) is included that may help you to understand the equivalent C++ code, but it loads the entire pattern file into RAM.
 
 ## How to use
 
-To minimize false positives, wait until Wikimedia releases a new data dump (usually at the third day of every month). You can get notified by subscribing to [the RSS feed](https://dumps.wikimedia.org/commonswiki/latest/commonswiki-latest-all-titles.gz-rss.xml).
+If you want to find invalid categories, you should wait until Wikimedia releases a new data dump (usually at the third day of every month) to minimize false positives. You can get notified by subscribing to [the RSS feed](https://dumps.wikimedia.org/commonswiki/latest/commonswiki-latest-all-titles.gz-rss.xml). If you want to find invalid files, you do not need to wait—the data dump is released every day.
+
+CommonsChecker4OSM has only been tested on Linux. It should work on Windows, but preparing the input files will be annoying. Step-by-step guide:
 
 1. Install [Osmium Tool](https://osmcode.org/osmium-tool/)
 2. Clone this repository
-3. Download required files into the same directory:
+3. Inside cloned repo, create `data` and `out` folders
+3. Download required files into the `data` directory:
    * `planet.pbf`: an Osmium-compatible [OpenStreetMap planet](https://wiki.openstreetmap.org/wiki/Planet.osm)
-   * [commonswiki-latest-all-titles.gz](https://dumps.wikimedia.org/commonswiki/latest/commonswiki-latest-all-titles.gz) (1.1 GB as of 2023)
-4. Choose an output format (see below). If you use a Unix-like operating system, you can run `make tsv` or `make geojson`. Otherwise, run the commands given below.
+4. Choose an output format (see below). Run either `make out/cats.tsv`, `make out/cats.geojson`, `make out/files.tsv`, or `make out/files.geojson` (output filename must be one of these hardcoded strings).
 
 ## Output formats
 
 ### Tab-separated values (TSV)
 
-Output contains invalid category names and the OpenStreetMap ID of the node, way, or relation with that category. This is the fastest format to produce. 
+Output contains OSM identifier (node, way, or relation ID) and `wikimedia_commons` value, separated by a tab. This is the fastest format to produce.
 
 ```console
-# Make sure you have 10min to spare
-$ osmium tags-filter -R planet.pbf 'nwr/wikimedia_commons=Category:*' -o commonscats-in-osm.xml
-$ python main.py out.tsv
-$ cat out.tsv
-n/1573735855	Conservatoire_National_de_V%C3%A9hicules_Historiques
-n/286133524	ref:sprockhoff No. 465
-n/3022117073	Wildwiesenwarte;Category:Views from the Wildwiesenwarte
-n/306593910	Prince George pub, Brighton Good pictures  Advanced...
-n/6426478285	Dorfkirche_Mechow_(Kyritz)?uselang=de
-w/297069904	https://commons.wikimedia.org/wiki/Category:Gr%C3%BCner_Graben_14_(G%C3%B6rlitz)
-w/320276921	 Ballyellen Upper Lock
-w/474166824	Nages-et-Solorgues#/media/File:Fontaine_Ranquet.jpg
-r/12931220	Brandenburger Straße 36;Riedelsberger Weg 2 (Bayreuth)
+$ make out/cats.tsv
+# osmium takes 10min for entire planet
+$ cat out/cats.tsv
+n/1573735855	Category:Conservatoire_National_de_V%C3%A9hicules_Historiques
+n/286133524	Category:ref:sprockhoff No. 465
+n/3022117073	Category:Wildwiesenwarte;Category:Views from the Wildwiesenwarte
+n/306593910	Category:Prince George pub, Brighton Good pictures  Advanced...
+n/6426478285	Category:Dorfkirche_Mechow_(Kyritz)?uselang=de
+w/297069904	Category:https://commons.wikimedia.org/wiki/Category:Gr%C3%BCner_Graben_14_(G%C3%B6rlitz)
+w/320276921	Category: Ballyellen Upper Lock
+w/474166824	Category:Nages-et-Solorgues#/media/File:Fontaine_Ranquet.jpg
+r/12931220	Category:Brandenburger Straße 36;Riedelsberger Weg 2 (Bayreuth)
 ```
 
 ### Line-by-Line GeoJSON
 
 [This format](https://learn.maproulette.org/documentation/line-by-line-geojson/) can be used to create a challenge on maproulette.org. It might lack a few categories that are present in the TSV format.
 
 ```console
-# Make sure you have 3GB RAM and 25min to spare
-$ osmium tags-filter -t planet.pbf 'nwr/wikimedia_commons=Category:*' -o planet-filtered.pbf
-$ osmium export planet-filtered.pbf -c config.json -o planet-filtered.geojson -f jsonseq
-$ python main.py out.geojson
-$ cat out.geojson
-{"type":"Feature","geometry":{"type":"Point","coordinates":[-2.0835284,53.3600557]},"properties":{"@type":"node","@id":29947059,"wikimedia_commons":"Category:Help Category:Middlewood railway station"}}
-{"type":"Feature","geometry":{"type":"LineString","coordinates":[[10.1212064,54.3247979],[10.120334100000001,54.3242942],[10.1192733,54.3236981],[10.1199922,54.3233703],[10.1204628,54.3231298],[10.1209243,54.3228965],[10.1222211,54.3236181],[10.1212064,54.3247979]]},"properties":{"@type":"way","@id":9408975,"wikimedia_commons":"Category:Wilhelmplatz (Kiel)"}}
-$ python to_maproulette.py out.geojson
+$ make out/cats.geojson
+# osmium takes 22min for entire planet
+$ cat cats.geojson
 {"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[-2.0835284,53.3600557]},"properties":{"@type":"node","@id":29947059,"wikimedia_commons":"Category:Help Category:Middlewood railway station"}}]}
 {"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"LineString","coordinates":[[10.1212064,54.3247979],[10.120334100000001,54.3242942],[10.1192733,54.3236981],[10.1199922,54.3233703],[10.1204628,54.3231298],[10.1209243,54.3228965],[10.1222211,54.3236181],[10.1212064,54.3247979]]},"properties":{"@type":"way","@id":9408975,"wikimedia_commons":"Category:Wilhelmplatz (Kiel)"}}]}
 ```
+
+## License
+
+CommonsChecker4OSM and BBHash are licensed under the MIT license:
+
+Copyright (c) 2015 Guillaume Rizk (BBHash)
+Copyright (c) 2023 Lennard Hofmann (CommonsChecker4OSM)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

main.py

@@ -1,60 +1,64 @@
 #!/usr/bin/env python3
-import xml.etree.ElementTree as ET
-import gzip
 import sys
 import json
 
-VALID_CATS = "commonscats-in-commons.txt"
-
-def load_valid_categories():
-    cats = set()
-    try:
-        with open(VALID_CATS) as f:
-            for line in f:
-                cats.add(line.rstrip('\n'))
-    except FileNotFoundError:
-        with gzip.open("commonswiki-latest-all-titles.gz", "rt") as f_in, open(VALID_CATS, "w") as f_out:
-            for line in f_in:
-                ns, category_name = line.split('\t', 1)
-                if ns == '14': # "Category:" namespace
-                    cats.add(category_name.rstrip('\n'))
-                    f_out.write(category_name)
-    return cats
-
-
-def tsv(cats, outfile):
-    with open(outfile, "w") as f:
-        tree = ET.parse("commonscats-in-osm.xml")
-        for elem in tree.findall(".//*tag[@k='wikimedia_commons']/.."):
-            wikimedia_commons = elem.find("tag[@k='wikimedia_commons']").get('v').removeprefix("Category:")
-            if wikimedia_commons.replace(' ', '_') not in cats:
-                f.write(f"{elem.tag[:1]}/{elem.get('id')}\t{wikimedia_commons}\n")
-
-
-def geojson(cats, outfile):
-    with open(outfile, "wb") as f:
-        for line in open("planet-filtered.geojson", "rb"):
-            wikimedia_commons = json.loads(line[1:])["properties"]["wikimedia_commons"]
-            if wikimedia_commons.removeprefix("Category:").replace(' ', '_') not in cats:
-                f.write(line)
+class InvalidCommonsTagDetector:
+    def __init__(self, prefix, keyfile):
+        self.prefix = prefix
+        with open(keyfile) as f:
+            self.titles = {line for line in f}
+
+    def detect(self, str):
+        return (str.startswith(self.prefix) and
+                str.removeprefix(self.prefix).replace(' ', '_') not in self.titles)
+
+
+def print_tsv(detector: InvalidCommonsTagDetector, tsvfile):
+    for line in open(tsvfile, "r"):
+        fields = line.split('\t')
+        if len(fields) != 2:
+            sys.stderr.write("malformed input line: " + line)
+            continue
+        if detector.detect(fields[1]):
+            sys.stdout.write(line)
+
+
+def print_geojson(detector: InvalidCommonsTagDetector, geojsonfile):
+    for line in open(geojsonfile, "rb"):
+        if detector.detect(json.loads(line[1:])["properties"]["wikimedia_commons"] + '\n'):
+            # MapRoulette-friendly output format:
+            sys.stdout.buffer.write(b'\x1e{"type":"FeatureCollection","features":[%s]}\n'
+                    % line[1:].rstrip(b'\n'))
 
 
 def usage():
-    print("""Usage: ./main.py outfile.geojson
-       ./main.py outfile.tsv""",
+    print("""Usage: ./main.py <prefix> <keyfile> <osmfile>
+
+Arguments:
+  <prefix>: namespace prefix (e.g. "Category:" or "File:")
+  <keyfile>:  list of MediaWiki page titles (one per line)
+  <osmfile>:  .tsv or .geojson file generated with make
+
+Example:
+   ./main.py Category: data/commonswiki-cats.txt data/planet-filtered.tsv""",
         file=sys.stderr)
     exit(1)
 
 
 def main():
-    if len(sys.argv) != 2:
+    if len(sys.argv) != 4:
         usage()
 
-    outfile = sys.argv[1]
-    if outfile.endswith(".tsv"):
-        tsv(load_valid_categories(), outfile)
-    elif outfile.endswith(".geojson"):
-        geojson(load_valid_categories(), outfile)
+    prefix = sys.argv[1]
+    keyfile = sys.argv[2]
+    osmfile = sys.argv[3]
+
+    if osmfile.endswith(".tsv"):
+        detector = InvalidCommonsTagDetector(prefix, keyfile)
+        print_tsv(detector, osmfile)
+    elif osmfile.endswith(".geojson"):
+        detector = InvalidCommonsTagDetector(prefix, keyfile)
+        print_geojson(detector, osmfile)
     else:
         usage()