New builtin function: geoip2_get_json

* src/builtin/geoip2.bi (geoip2_get): Throw e_range if the lookup
path does not exist in the returned data.
Return empty string if there is no data in the entry.
(geoip2_get_json): New function.
* src/prog.c (heap_obstack_vsprintf): Don't overwrite previously
written data.
* doc/functions.texi: Describe the geoip2 functions.
* NEWS: Update.

5 files changed, 399 insertions, 16 deletions

diff --git a/NEWS b/NEWS
index 55652c10..7f2bb6fb 100644
--- a/NEWS
+++ b/NEWS
@@ -1,4 +1,4 @@
-Mailfromd NEWS -- history of user-visible changes. 2020-11-04
+Mailfromd NEWS -- history of user-visible changes. 2020-11-08
 See the end of file for copying conditions.
 
 Please send Mailfromd bug reports to <bug-mailfromd@gnu.org.ua>
@@ -67,6 +67,51 @@ Internet Domain Name System.  Example usage:
 These used to be built-in functions.  Starting from this release, they
 are implemented in pure MFL, in the module 'dns'.  Be sure to require
 this module if you are using these functions.
+
+* New geolocation functions (libmaxminddb)
+
+The support for geolocation using the libmaxminddb library is added.
+It is enabled if the libmaxminddb is installed and can be located
+using pkg-config.  The new configure option `--with-geoip2' is available
+to expressly request it.
+
+If enabled, the GeoIP2 support is indicated by the following line in
+the output of configure:
+
+  Enable GeoIP2 support..................... yes
+
+In the output of `mailfromd --show-defaults', it is indicated by the
+word GeoIP2 in the list of optional features.
+
+The preprocessor macro WITH_GEOIP2 is defined if the GeoIP2 support is
+compiled in.
+
+The following new functions are available:
+
+** void geoip2_open (string FILENAME)
+
+Opens the geolocation database file FILENAME.
+
+** string geoip2_dbname (void)
+
+Returns the name of the geolocation database currently in use.
+
+** string geoip2_get (string IP, string PATH)
+
+Looks up the ip address IP in the database and returns the data item
+identified by PATH.  E.g. to retrieve the country code:
+
+   geoip2_get($client_addr, 'country.iso_code')
+
+** string geoip2_get_json (string IP; number INDENT)
+
+Looks up IP in the database and returns entire data set, formatted as
+JSON object.
+
+* Legacy geolocation functions
+
+Support for the legacy geolocation library libGeoIP is maked as
+deprecated.  Users are advised to migrate to GeoIP2 instead.
 
 Version 8.8, 2020-07-26
 
diff --git a/configure.ac b/configure.ac
index becdc956..f3934956 100644
--- a/configure.ac
+++ b/configure.ac
@@ -540,7 +540,7 @@ AM_CONDITIONAL([PMULT_COND], [test "$enable_pmilter" = yes])
 # GeoIP
 AC_ARG_WITH([geoip],
             AC_HELP_STRING([--with-geoip],
-                           [use GeoIP library]),
+                           [use the legacy GeoIP library (DEPRECATED)]),
             [
 case "${withval}" in
   yes) status_geoip=yes ;;
@@ -566,7 +566,7 @@ fi
 
 # GeoIP2
 AC_ARG_WITH([geoip2],
-            AC_HELP_STRING([--with-geoip],
+            AC_HELP_STRING([--with-geoip2],
                            [use MaxMind GeoIP2 library]),
             [
 case "${withval}" in
diff --git a/doc/functions.texi b/doc/functions.texi
index cb58718e..0477e795 100644
--- a/doc/functions.texi
+++ b/doc/functions.texi
@@ -2491,19 +2491,164 @@ data types are implemented.
 @node Geolocation functions
 @section Geolocation functions
 @cindex geolocation
+@cindex GeoIP2
+@flindex libmaxminddb
+@kwindex WITH_GEOIP2
+  The @dfn{geolocation functions} allow you to identify the country where
+the given IP address or host name is located.  These functions are
+available only if the @code{libmaxminddb} library is installed and
+@command{mailfromd} is compiled with the @samp{GeoIP2} support.
+
+  The @code{libmaxminddb} library is distributed by @samp{MaxMind} under
+the terms of the @cite{Apache License} Version 2.0.  It is available
+from @uref{https://dev.maxmind.com/geoip/geoip2/downloadable/#MaxMind_APIs}.
+
+  Historically, @command{mailfromd} supports also the legacy
+@samp{GeoIP} library.  If you are interested in it, please refer to
+@ref{Legacy geoip support}.
+
+@deftypefn {Built-in Function} void geoip2_open (string @var{filename})
+Opens the geolocation database file @var{filename}.  The database must
+be in GeoIP2 format.
+
+If the database cannot be opened, @code{geoip2_open} throws the
+@code{e_failure} exception.
+
+If this function is not called, geolocation functions described below
+will try to open the database file @samp{/usr/share/GeoIP/GeoLite2-City.mmdb}.
+@end deftypefn
+
+@deftypefn {Built-in Function} string geoip2_dbname (void)
+Returns the name of the geolocation database currently in use.
+@end deftypefn
+
+The geolocation database for each IP address, which serves as a look
+up key, stores a set of items describing this IP.  This set is
+organized as a map of key-value pairs.  Each key is a string value.
+A value can be a scalar, another map or array of values.  Using
+JSON notation, the result of a look up in the database might look as:
+
+@example
+@group
+@{
+    "country":@{
+        "geoname_id":2921044,
+        "iso_code":"DE",
+        "names":@{
+            "en": "Germany",
+            "de": "Deutschland",
+            "fr":"Allemagne"
+        @},
+    @},
+    "continent":@{
+        "code":"EU",
+        "geoname_id":6255148,
+        "names":@{
+            "en":"Europe",
+            "de":"Europa",
+            "fr":"Europe"
+        @}
+    @},
+    "location":@{
+        "accuracy_radius":200,
+        "latitude":49.4478,
+        "longitude":11.0683,
+        "time_zone":"Europe/Berlin"
+    @},
+    "city":@{ 
+        "geoname_id":2861650,
+        "names":@{
+            "en":"Nuremberg",
+            "de":"N@"urnberg",
+            "fr":"Nuremberg"
+        @}
+    @},
+    "subdivisions":[@{
+        "geoname_id":2951839,
+        "iso_code":"BY",
+        "names":@{
+            "en":"Bavaria",
+            "de":"Bayern",
+            "fr":"Bavi@`ere"
+        @}
+    @}
+@}
+@end group
+@end example
+
+Each particular data item in such structure is identified by its
+@dfn{search path}, which is a dot-delimited list of key names leading
+to that value.  For example, using the above map, the name of the city
+in English can be retrieved using the key @code{city.names.en}.
+
+@deftypefn {Built-in Function} string geoip2_get (string @var{ip}, string @var{path})
+Looks up the IP address @var{ip} in the geolocation database.  If
+found, returns data item identified by the search path @var{path}.
+
+The function can throw the following exceptions:
+
+@table @asis
+@item e_not_found
+The @var{ip} was not found in the database.
+
+@item e_range
+The @var{path} does not exist the returned map.
+
+@item e_failure
+General error occurred.  E.g. the database cannot be opened, @var{ip}
+is not a valid IP address, etc.
+@end table
+@end deftypefn
+
+@deftypefn {Built-in Function} string geoip2_get_json (string @var{ip} [; number @var{indent})
+Looks up the @var{ip} in the database and returns entire data set
+associated with it, formatted as a JSON object.  If the optional
+parameter @var{indent} is supplied and is greater than zero, it gives
+the indentation for each nesting level in the JSON object.
+@end deftypefn
+
+@vrindex WITH_GEOIP2
+Applications may test whether the GeoIP2 support is present and
+enable the corresponding code blocks conditionally, by testing if 
+the @samp{WITH_GEOIP2} m4 macro is defined.  For example, the
+following code adds to the message the @samp{X-Originator-Country}
+header, containing the 2 letter code of the country where the client
+machine is located.  If @command{mailfromd} is compiled without
+@samp{GeoIP} support, it does nothing:
+
+@example
+m4_ifdef(`WITH_GEOIP2',`
+  try
+  do
+    header_add("X-Originator-Country", geoip2_get($client_addr,
+                                                  'country.iso_code'))
+  done
+  catch e_not_found or e_range
+  do
+    pass
+  done
+')
+@end example
+
+@node Legacy geoip support
+@subsection Legacy geoip support
 @cindex GeoIP
 @flindex libGeoIP
 @kwindex WITH_GEOIP
-  The @dfn{geolocation functions} allow you to identify the country where
-the given IP address or host name is located.  These functions are
-available only if the @samp{GeoIP} library is installed and
-@command{mailfromd} is compiled with the @samp{GeoIP} support.  The
-@command{m4} macro @samp{WITH_GEOIP} is defined if it is so.
 
-The @file{GeoIP} is a geolocational package distributed by
-@samp{MaxMind} under the terms of the GNU Lesser General Public
-License.  The library is available from
-@uref{http://www.maxmind.com/app/c}.
+For compatibility with older releases, @command{mailfromd} supports
+the legacy @samp{GeoIP} library.  This support is going to be removed
+in the next release, so its use is not recommended.  Please use
+@samp{GeoIP2} instead.
+
+The support for the legacy @file{GeoIP} library is available if 
+@command{mailfromd} is compiled with the @samp{GeoIP} support (the
+@option{--with-geoip} configure option).  The @command{m4} macro
+@samp{WITH_GEOIP} is defined if it is so.
+
+The legacy @file{GeoIP} is distributed by @samp{MaxMind} under the
+terms of the GNU Lesser General Public License.  The library is
+available from @uref{http://www.maxmind.com/app/c}.
 
 @deftypefn {Built-in Function} string geoip_country_code_by_addr (@
     string @var{ip} [, bool @var{tlc}])
@@ -2526,7 +2671,7 @@ exception.
 
 @vrindex WITH_GEOIP
 Applications may test whether the GeoIP support is present and
-enable corresponding code blocks conditionally by testing if 
+enable corresponding code blocks conditionally, by testing if 
 the @samp{WITH_GEOIP} m4 macro is defined.  For example, the
 following code adds to the message the @samp{X-Originator-Country}
 header, containing the 2 letter code of the country where the client
diff --git a/src/builtin/geoip2.bi b/src/builtin/geoip2.bi
index 0f8700ea..ad790342 100644
--- a/src/builtin/geoip2.bi
+++ b/src/builtin/geoip2.bi
@@ -182,12 +182,17 @@ MF_DEFUN(geoip2_get, STRING, STRING ip, STRING pathstr)
 	rc = MMDB_aget_value(&result.entry, &entry_data,
 			     (const char * const* const) ws.ws_wordv);
 	mu_wordsplit_free(&ws);
-	
+
 	MF_ASSERT(rc == MMDB_SUCCESS,
-		  mfe_failure,
+		  (rc == MMDB_LOOKUP_PATH_DOES_NOT_MATCH_DATA_ERROR ||
+		   rc == MMDB_INVALID_LOOKUP_PATH_ERROR)
+		     ? mfe_range : mfe_failure,
 		  "%s %s: MMDB_aget_value %s",
 		  pathstr, ip, MMDB_strerror(rc));
 
+	if (!entry_data.has_data)
+		MF_RETURN("");
+	
 	MF_ASSERT(entry_data.type >= 0
 		  && entry_data.type <= sizeof (entry_conv) / sizeof (entry_conv[0])
 		  && entry_conv[entry_data.type],
@@ -202,5 +207,193 @@ MF_DEFUN(geoip2_get, STRING, STRING ip, STRING pathstr)
 	MF_RETURN_OBSTACK();	
 }
 END
+
+struct object_type {
+	enum { OBJ_MAP, OBJ_ARRAY } type;
+	size_t count;
+	unsigned level;
+	struct object_type *prev;
+};
 
+static inline void
+object_type_push(struct object_type **otp, int type, size_t count)
+{
+	struct object_type *t = mu_alloc(sizeof(*t));
+	t->type = type;
+	t->count = count;
+	t->prev = *otp;
+	t->level = t->prev ? t->prev->level + 1 : 1;
+	*otp = t;
+}
+
+static inline void
+object_type_pop(struct object_type **otp)
+{
+	struct object_type *t = *otp;
+	*otp = t->prev;
+	free(t);
+}
 
+MF_DEFUN(geoip2_get_json, STRING, STRING ip, OPTIONAL, NUMBER indent)
+{
+	struct geoip2_storage *gs = geoip2_open(env);
+	MMDB_lookup_result_s result;
+        int mmdb_error;
+	int gai_error;
+	int rc;
+	MMDB_entry_data_list_s *data_list, *p;
+	struct object_type *type = NULL;
+	int iskey = 1;
+	size_t i;
+	char *indent_str = NULL;
+	
+	result = MMDB_lookup_string(&gs->db, ip, &gai_error, &mmdb_error);
+	MF_ASSERT(gai_error == 0,
+		  mfe_failure,
+		  "%s: %s",
+		  ip, gai_strerror(gai_error));
+	
+	MF_ASSERT(mmdb_error == MMDB_SUCCESS,
+		  mfe_failure,
+		  "%s: %s",
+		  ip, MMDB_strerror(mmdb_error));
+
+	MF_ASSERT(result.found_entry != 0,
+		  mfe_not_found,
+		  _("IP not found in the database"));
+
+	rc = MMDB_get_entry_data_list(&result.entry, &data_list);
+	MF_ASSERT(rc == MMDB_SUCCESS,
+		  mfe_failure,
+		  "%s: MMDB_aget_value %s",
+		  ip, MMDB_strerror(rc));
+
+	/*MMDB_dump_entry_data_list(stdout, data_list, 4);*/
+
+	indent = MF_OPTVAL(indent, 0);
+	if (indent) {
+		indent_str = mu_alloc(indent+1);
+		memset(indent_str, ' ', indent);
+		indent_str[indent] = 0;
+	}
+	
+	MF_OBSTACK_BEGIN();
+	for (p = data_list; p; p = p->next) {
+		if (iskey && type && indent_str) {
+			MF_OBSTACK_1GROW('\n');
+			for (i = 0; i < type->level; i++)
+				MF_OBSTACK_GROW(indent_str);
+		}
+
+		if (type == NULL && p->entry_data.type != MMDB_DATA_TYPE_MAP) {
+			MF_OBSTACK_CANCEL();
+			free(indent_str);
+			MMDB_free_entry_data_list(data_list);
+			MF_THROW(mfe_failure,
+				 "%s",
+				 _("malformed data list"));
+		}
+		
+		switch (p->entry_data.type) {
+		case MMDB_DATA_TYPE_MAP:
+			object_type_push(&type, OBJ_MAP, p->entry_data.data_size);
+			MF_OBSTACK_1GROW('{');
+			iskey = 1;
+			continue;
+			
+		case MMDB_DATA_TYPE_ARRAY:
+			object_type_push(&type, OBJ_ARRAY, p->entry_data.data_size);
+			MF_OBSTACK_1GROW('[');
+			continue;
+			
+		case MMDB_DATA_TYPE_UTF8_STRING:
+			MF_OBSTACK_1GROW('"');
+			MF_OBSTACK_GROW((char*)p->entry_data.utf8_string, p->entry_data.data_size);
+			MF_OBSTACK_1GROW('"');
+			if (iskey)
+				MF_OBSTACK_1GROW(':');
+			break;
+			
+		case MMDB_DATA_TYPE_DOUBLE:
+			MF_OBSTACK_PRINTF("%g", p->entry_data.double_value);
+			break;
+			
+		case MMDB_DATA_TYPE_BYTES:
+			MF_OBSTACK_1GROW('[');
+			for (i = 0; i < p->entry_data.data_size; i++) {
+				if (i) MF_OBSTACK_1GROW([<','>]);
+				MF_OBSTACK_GROW("%d", p->entry_data.bytes[i]);
+			}
+			MF_OBSTACK_1GROW(']');
+			break;
+			
+		case MMDB_DATA_TYPE_UINT16:
+			MF_OBSTACK_PRINTF("%u", p->entry_data.uint16);
+			break;
+			
+		case MMDB_DATA_TYPE_UINT32:
+			MF_OBSTACK_PRINTF("%" PRIu32, p->entry_data.uint32);
+			break;
+			
+		case MMDB_DATA_TYPE_INT32:
+			MF_OBSTACK_PRINTF("%" PRIi32, p->entry_data.int32);
+			break;
+			
+		case MMDB_DATA_TYPE_UINT64:
+			MF_OBSTACK_PRINTF("%" PRIu64, p->entry_data.uint64);
+			break;
+			
+		case MMDB_DATA_TYPE_UINT128:
+			MF_OBSTACK_GROW("'N/A");//FIXME
+			break;
+			
+		case MMDB_DATA_TYPE_BOOLEAN:
+			MF_OBSTACK_GROW(p->entry_data.boolean ? "true" : "false");
+			break;
+					
+		case MMDB_DATA_TYPE_FLOAT:
+			MF_OBSTACK_PRINTF("%g", p->entry_data.float_value);
+			break;
+
+		default:
+			MF_OBSTACK_CANCEL();
+			free(indent_str);
+			MMDB_free_entry_data_list(data_list);
+			MF_THROW(mfe_failure,
+				 _("unsupported MMDB data type %d"),
+				 p->entry_data.type);
+		}
+
+		if (type) {
+			iskey = !iskey;
+			if (iskey == 1) {
+				while (type && --type->count == 0) {
+					if (indent_str) {
+						MF_OBSTACK_1GROW('\n');
+						for (i = 1; i < type->level; i++)
+							MF_OBSTACK_GROW(indent_str);
+					}
+					MF_OBSTACK_1GROW(type->type == OBJ_MAP
+							 ? '}' : ']');
+					object_type_pop(&type);
+				}
+				if (type) {
+					MF_OBSTACK_1GROW([<','>]);
+				}
+			}
+		}
+	}
+	MF_OBSTACK_1GROW(0);
+	free(indent_str);
+	MMDB_free_entry_data_list(data_list);
+
+	if (type) {
+		while (type)
+			object_type_pop(&type);
+		MF_THROW(mfe_failure,
+			 _("malformed data list: reported and actual number of keys differ"));
+	}
+	
+	MF_RETURN_OBSTACK();	
+}
+END
diff --git a/src/prog.c b/src/prog.c
index f9d1494a..bd499353 100644
--- a/src/prog.c
+++ b/src/prog.c
@@ -861,7 +861,7 @@ heap_obstack_vsprintf(eval_environ_t env, const char *fmt, va_list ap)
 
 		size = heap_obstack_size(env);
 		va_copy(apc, ap);
-		n = vsnprintf((char*) env_data_ref(env, env->temp_start),
+		n = vsnprintf((char*) env_data_ref(env, env->temp_start) + env->temp_size,
 			      size, fmt, apc);
 		va_end(apc);
 		if (n >= size) {