summaryrefslogtreecommitdiffabout
authorSergey Poznyakoff <gray@gnu.org>2018-05-24 08:35:24 (GMT)
committer Sergey Poznyakoff <gray@gnu.org>2018-05-24 08:35:24 (GMT)
commit371bb85fe378ffd0ed6ddc81985d450cef5835a3 (patch) (unidiff)
tree6ab2d142b7aa1f9b29ec50b891c8293774964209
parentf82d0b213fd2cdce80dc891906fba8d589115664 (diff)
downloadgdbm-371bb85fe378ffd0ed6ddc81985d450cef5835a3.tar.gz
gdbm-371bb85fe378ffd0ed6ddc81985d450cef5835a3.tar.bz2
More error checking; improve gdbm_recover
* Makefile.am (set-dist-date): New rule (dist-hook): Catch FIXMEs in NEWS. * NEWS: Updated. * src/findkey.c (gdbm_bucket_element_valid_p): New function. (_gdbm_read_entry): Validate the retrieved bucket element. * src/gdbm.h.in (gdbm_recovery): New member: duplicate_keys. (GDBM_BAD_HASH_TABLE): New error code. * src/gdbmdefs.h (TYPE_WIDTH,SIGNED_TYPE_MAXIMUM) (OFF_T_MAX): New defines. (off_t_sum_ok): New function. (gdbm_bucket_element_valid_p): New prototype. * src/gdbmerrno.c: Support for GDBM_BAD_HASH_TABLE code. * src/gdbmtool.c (recover_handler): Fix argument counting. New argument 'summary' prints statistics summary at the end of the run. (export_handler,import_handler): Fix argument counting. * src/mmap.c (SUM_FILE_SIZE): Rewrite as inlined function. Add error checking. (_gdbm_mapped_remap): More error checking. * src/recover.c (run_recovery): Don't bail out on GDBM_CANNOT_REPLACE. (gdbm_recover): Initialize duplicate_keys * src/systems.h: Include limits.h
Diffstat (more/less context) (ignore whitespace changes)
-rw-r--r--Makefile.am16
-rw-r--r--NEWS31
-rw-r--r--src/findkey.c24
-rw-r--r--src/gdbm.h.in4
-rw-r--r--src/gdbmdefs.h17
-rw-r--r--src/gdbmerrno.c3
-rw-r--r--src/gdbmtool.c27
-rw-r--r--src/mmap.c22
-rw-r--r--src/recover.c32
-rw-r--r--src/systems.h1
10 files changed, 157 insertions, 20 deletions
diff --git a/Makefile.am b/Makefile.am
index 6e00dec..ecc03df 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -24,12 +24,26 @@ SUBDIRS = po src doc $(MAYBE_COMPAT) tests
24 24
25AM_DISTCHECK_CONFIGURE_FLAGS = --enable-libgdbm-compat 25AM_DISTCHECK_CONFIGURE_FLAGS = --enable-libgdbm-compat
26 26
27dist-hook: 27.PHONY: set-dist-date
28set-dist-date:
28 rm -f $(distdir)/src/version.c; \ 29 rm -f $(distdir)/src/version.c; \
29 d=`date '+%d/%m/%Y'`; \ 30 d=`date '+%d/%m/%Y'`; \
30 sed 's|/\*@DIST_DATE@\*/|"'"$$d"'"|' $(srcdir)/src/version.c > \ 31 sed 's|/\*@DIST_DATE@\*/|"'"$$d"'"|' $(srcdir)/src/version.c > \
31 $(distdir)/src/version.c 32 $(distdir)/src/version.c
32 33
34dist-hook: ChangeLog set-dist-date
35 @if test -f ChangeLog && test -f NEWS; then \
36 PATCHLEV=`echo "$(PACKAGE_VERSION)" | sed -r "s/[0-9]+\.[0-9]+\.?//"`;\
37 if test $${PATCHLEV:-0} -lt 50; then \
38 if grep -q FIXME NEWS; then \
39 echo >&2 "*** NEWS file contains FIXMEs"; \
40 echo >&2 "*** Aborting"; \
41 exit 1; \
42 fi; \
43 fi; \
44 fi
45
46
33gen_start_date = 2016-07-08 47gen_start_date = 2016-07-08
34prev_change_log = ChangeLog.cvs 48prev_change_log = ChangeLog.cvs
35 49
diff --git a/NEWS b/NEWS
index e004eec..24180d8 100644
--- a/NEWS
+++ b/NEWS
@@ -1,4 +1,4 @@
1GNU dbm NEWS -- history of user-visible changes. 2018-05-19 1GNU dbm NEWS -- history of user-visible changes. 2018-05-24
2Copyright (C) 1990-2018 Free Software Foundation, Inc. 2Copyright (C) 1990-2018 Free Software Foundation, Inc.
3See the end of file for copying conditions. 3See the end of file for copying conditions.
4 4
@@ -6,12 +6,41 @@ Please send gdbm bug reports to <bug-gdbm@gnu.org>.
6 6
7Version 1.14.90 7Version 1.14.90
8 8
9FIXME: BUMP VI_MAJOR
10
9* Implement database consistency checks 11* Implement database consistency checks
10 12
11* Improved error checking 13* Improved error checking
12 14
13* Removed gdbm-1.8.3 compatibility layer 15* Removed gdbm-1.8.3 compatibility layer
14 16
17* Commands can be given to gdbmtool in the command line
18
19The syntax is:
20
21 gdbmtool DBNAME COMMAND [ARGS...]
22
23Multiple commands are separated by semicolon (take care to escape it),
24e.g.:
25
26 gdbmtool t.db count\; avail
27
28* Fixed data conversion bugs in storing structured keys or content
29
30* New member in the gdbm_recovery structure: duplicate_keys.
31
32Upon return from gdbm_recover, this member holds the number of keys
33that were not recovered, because the same key has already been stored
34in the database. The actual number of stored keys is thus
35recovered_keys - duplicate_keys.
36
37* New error codes.
38
39 GDBM_BAD_BUCKET "Malformed bucket header"
40 GDBM_BAD_HEADER "Malformed database file header"
41 GDBM_BAD_AVAIL "Malformed avail_block"
42 GDBM_BAD_HASH_TABLE "Malformed hash table"
43
15 44
16Version 1.14.1 - 2018-01-03 45Version 1.14.1 - 2018-01-03
17 46
diff --git a/src/findkey.c b/src/findkey.c
index 7638b04..bd9fd83 100644
--- a/src/findkey.c
+++ b/src/findkey.c
@@ -22,7 +22,21 @@
22 22
23#include "gdbmdefs.h" 23#include "gdbmdefs.h"
24 24
25 25int
26gdbm_bucket_element_valid_p (GDBM_FILE dbf, int elem_loc)
27{
28 return
29 elem_loc < dbf->header->bucket_elems
30 && dbf->bucket->h_table[elem_loc].hash_value != -1
31 && dbf->bucket->h_table[elem_loc].key_size >= 0
32 && off_t_sum_ok (dbf->bucket->h_table[elem_loc].data_pointer,
33 dbf->bucket->h_table[elem_loc].key_size)
34 && dbf->bucket->h_table[elem_loc].data_size >= 0
35 && off_t_sum_ok (dbf->bucket->h_table[elem_loc].data_pointer
36 + dbf->bucket->h_table[elem_loc].key_size,
37 dbf->bucket->h_table[elem_loc].data_size);
38}
39
26/* Read the data found in bucket entry ELEM_LOC in file DBF and 40/* Read the data found in bucket entry ELEM_LOC in file DBF and
27 return a pointer to it. Also, cache the read value. */ 41 return a pointer to it. Also, cache the read value. */
28 42
@@ -34,11 +48,17 @@ _gdbm_read_entry (GDBM_FILE dbf, int elem_loc)
34 int data_size; 48 int data_size;
35 off_t file_pos; 49 off_t file_pos;
36 data_cache_elem *data_ca; 50 data_cache_elem *data_ca;
37 51
38 /* Is it already in the cache? */ 52 /* Is it already in the cache? */
39 if (dbf->cache_entry->ca_data.elem_loc == elem_loc) 53 if (dbf->cache_entry->ca_data.elem_loc == elem_loc)
40 return dbf->cache_entry->ca_data.dptr; 54 return dbf->cache_entry->ca_data.dptr;
41 55
56 if (!gdbm_bucket_element_valid_p (dbf, elem_loc))
57 {
58 GDBM_SET_ERRNO (dbf, GDBM_BAD_HASH_TABLE, TRUE);
59 return NULL;
60 }
61
42 /* Set sizes and pointers. */ 62 /* Set sizes and pointers. */
43 key_size = dbf->bucket->h_table[elem_loc].key_size; 63 key_size = dbf->bucket->h_table[elem_loc].key_size;
44 data_size = dbf->bucket->h_table[elem_loc].data_size; 64 data_size = dbf->bucket->h_table[elem_loc].data_size;
diff --git a/src/gdbm.h.in b/src/gdbm.h.in
index 61d5707..e576c69 100644
--- a/src/gdbm.h.in
+++ b/src/gdbm.h.in
@@ -152,6 +152,7 @@ typedef struct gdbm_recovery_s
152 size_t recovered_buckets; 152 size_t recovered_buckets;
153 size_t failed_keys; 153 size_t failed_keys;
154 size_t failed_buckets; 154 size_t failed_buckets;
155 size_t duplicate_keys;
155 char *backup_name; 156 char *backup_name;
156} gdbm_recovery; 157} gdbm_recovery;
157 158
@@ -222,9 +223,10 @@ extern int gdbm_copy_meta (GDBM_FILE dst, GDBM_FILE src);
222# define GDBM_BAD_BUCKET 32 223# define GDBM_BAD_BUCKET 32
223# define GDBM_BAD_HEADER 33 224# define GDBM_BAD_HEADER 33
224# define GDBM_BAD_AVAIL 34 225# define GDBM_BAD_AVAIL 34
226# define GDBM_BAD_HASH_TABLE 35
225 227
226 # define _GDBM_MIN_ERRNO0 228 # define _GDBM_MIN_ERRNO0
227 # define _GDBM_MAX_ERRNOGDBM_BAD_AVAIL 229 # define _GDBM_MAX_ERRNOGDBM_BAD_HASH_TABLE
228 230
229/* This one was never used and will be removed in the future */ 231/* This one was never used and will be removed in the future */
230# define GDBM_UNKNOWN_UPDATE GDBM_UNKNOWN_ERROR 232# define GDBM_UNKNOWN_UPDATE GDBM_UNKNOWN_ERROR
diff --git a/src/gdbmdefs.h b/src/gdbmdefs.h
index 5305b0d..1bb519b 100644
--- a/src/gdbmdefs.h
+++ b/src/gdbmdefs.h
@@ -26,6 +26,22 @@
26#define _(s) gettext (s) 26#define _(s) gettext (s)
27#define N_(s) s 27#define N_(s) s
28 28
29/* The width in bits of the integer type or expression T. */
30#define TYPE_WIDTH(t) (sizeof (t) * CHAR_BIT)
31
32#define SIGNED_TYPE_MAXIMUM(t) \
33 ((t) ((((t) 1 << (TYPE_WIDTH (t) - 2)) - 1) * 2 + 1))
34
35/* Maximum value for off_t */
36#define OFF_T_MAX SIGNED_TYPE_MAXIMUM (off_t)
37
38/* Return true if A can be added to B without integer overflow */
39static inline off_t
40off_t_sum_ok (off_t a, off_t b)
41{
42 return OFF_T_MAX - a >= b;
43}
44
29/* The type definitions are next. */ 45/* The type definitions are next. */
30 46
31/* The available file space is stored in an "avail" table. The one with 47/* The available file space is stored in an "avail" table. The one with
@@ -93,6 +109,7 @@ typedef struct
93 int data_size; /* Size of associated data in the file. */ 109 int data_size; /* Size of associated data in the file. */
94} bucket_element; 110} bucket_element;
95 111
112extern int gdbm_bucket_element_valid_p (GDBM_FILE dbf, int elem_loc);
96 113
97/* A bucket is a small hash table. This one consists of a number of 114/* A bucket is a small hash table. This one consists of a number of
98 bucket elements plus some bookkeeping fields. The number of elements 115 bucket elements plus some bookkeeping fields. The number of elements
diff --git a/src/gdbmerrno.c b/src/gdbmerrno.c
index 896bf70..52cfe30 100644
--- a/src/gdbmerrno.c
+++ b/src/gdbmerrno.c
@@ -134,7 +134,8 @@ const char * const gdbm_errlist[_GDBM_MAX_ERRNO+1] = {
134 [GDBM_DIR_OVERFLOW] = N_("Bucket directory overflow"), 134 [GDBM_DIR_OVERFLOW] = N_("Bucket directory overflow"),
135 [GDBM_BAD_BUCKET] = N_("Malformed bucket header"), 135 [GDBM_BAD_BUCKET] = N_("Malformed bucket header"),
136 [GDBM_BAD_HEADER] = N_("Malformed database file header"), 136 [GDBM_BAD_HEADER] = N_("Malformed database file header"),
137 [GDBM_BAD_AVAIL] = N_("Malforemd avail_block") 137 [GDBM_BAD_AVAIL] = N_("Malformed avail_block"),
138 [GDBM_BAD_HASH_TABLE] = N_("Malformed hash table")
138}; 139};
139 140
140const char * 141const char *
diff --git a/src/gdbmtool.c b/src/gdbmtool.c
index 33bdf93..9c6eebe 100644
--- a/src/gdbmtool.c
+++ b/src/gdbmtool.c
@@ -552,7 +552,7 @@ err_printer (void *data GDBM_ARG_UNUSED, char const *fmt, ...)
552 fprintf (stderr, "\n"); 552 fprintf (stderr, "\n");
553} 553}
554 554
555/* recover verbose backup max-failed-keys=N max-failed-buckets=N max-failures=N */ 555/* recover sumamry verbose backup max-failed-keys=N max-failed-buckets=N max-failures=N */
556void 556void
557recover_handler (struct handler_param *param) 557recover_handler (struct handler_param *param)
558{ 558{
@@ -561,8 +561,9 @@ recover_handler (struct handler_param *param)
561 int rc; 561 int rc;
562 int i; 562 int i;
563 char *p; 563 char *p;
564 int summary = 0;
564 565
565 for (i = 1; i < param->argc; i++) 566 for (i = 0; i < param->argc; i++)
566 { 567 {
567 char *arg = PARAM_STRING (param, i); 568 char *arg = PARAM_STRING (param, i);
568 if (strcmp (arg, "verbose") == 0) 569 if (strcmp (arg, "verbose") == 0)
@@ -570,6 +571,10 @@ recover_handler (struct handler_param *param)
570 rcvr.errfun = err_printer; 571 rcvr.errfun = err_printer;
571 flags |= GDBM_RCVR_ERRFUN; 572 flags |= GDBM_RCVR_ERRFUN;
572 } 573 }
574 else if (strcmp (arg, "summary") == 0)
575 {
576 summary = 1;
577 }
573 else if (strcmp (arg, "backup") == 0) 578 else if (strcmp (arg, "backup") == 0)
574 { 579 {
575 rcvr.errfun = err_printer; 580 rcvr.errfun = err_printer;
@@ -617,6 +622,19 @@ recover_handler (struct handler_param *param)
617 if (rc == 0) 622 if (rc == 0)
618 { 623 {
619 fprintf (param->fp, _("Recovery succeeded.\n")); 624 fprintf (param->fp, _("Recovery succeeded.\n"));
625 if (summary)
626 {
627 fprintf (param->fp,
628 _("Keys recovered: %lu, failed: %lu, duplicate: %lu\n"),
629 (unsigned long) rcvr.recovered_keys,
630 (unsigned long) rcvr.failed_keys,
631 (unsigned long) rcvr.duplicate_keys);
632 fprintf (param->fp,
633 _("Buckets recovered: %lu, failed: %lu\n"),
634 (unsigned long) rcvr.recovered_buckets,
635 (unsigned long) rcvr.failed_buckets);
636 }
637
620 if (rcvr.backup_name) 638 if (rcvr.backup_name)
621 { 639 {
622 fprintf (param->fp, 640 fprintf (param->fp,
@@ -925,7 +943,7 @@ export_handler (struct handler_param *param)
925 int i; 943 int i;
926 int filemode; 944 int filemode;
927 945
928 for (i = 1; i < param->argc; i++) 946 for (i = 0; i < param->argc; i++)
929 { 947 {
930 if (strcmp (PARAM_STRING (param, i), "truncate") == 0) 948 if (strcmp (PARAM_STRING (param, i), "truncate") == 0)
931 flags = GDBM_NEWDB; 949 flags = GDBM_NEWDB;
@@ -959,7 +977,7 @@ import_handler (struct handler_param *param)
959 int i; 977 int i;
960 int rc; 978 int rc;
961 979
962 for (i = 1; i < param->argc; i++) 980 for (i = 0; i < param->argc; i++)
963 { 981 {
964 if (strcmp (PARAM_STRING (param, i), "replace") == 0) 982 if (strcmp (PARAM_STRING (param, i), "replace") == 0)
965 flag = GDBM_REPLACE; 983 flag = GDBM_REPLACE;
@@ -1220,6 +1238,7 @@ struct command command_tab[] = {
1220 { S(recover), T_CMD, 1238 { S(recover), T_CMD,
1221 checkdb, recover_handler, NULL, 1239 checkdb, recover_handler, NULL,
1222 { { "[verbose]", GDBM_ARG_STRING }, 1240 { { "[verbose]", GDBM_ARG_STRING },
1241 { "[summary]", GDBM_ARG_STRING },
1223 { "[backup]", GDBM_ARG_STRING }, 1242 { "[backup]", GDBM_ARG_STRING },
1224 { "[max-failed-keys=N]", GDBM_ARG_STRING }, 1243 { "[max-failed-keys=N]", GDBM_ARG_STRING },
1225 { "[max-failed-buckets=N]", GDBM_ARG_STRING }, 1244 { "[max-failed-buckets=N]", GDBM_ARG_STRING },
diff --git a/src/mmap.c b/src/mmap.c
index 114d8b2..24ede29 100644
--- a/src/mmap.c
+++ b/src/mmap.c
@@ -44,8 +44,15 @@
44# define _GDBM_NEED_REMAP(dbf) \ 44# define _GDBM_NEED_REMAP(dbf) \
45 (!(dbf)->mapped_region || (dbf)->mapped_pos == (dbf)->mapped_size) 45 (!(dbf)->mapped_region || (dbf)->mapped_pos == (dbf)->mapped_size)
46/* Return the sum of the currently mapped size and DELTA */ 46/* Return the sum of the currently mapped size and DELTA */
47# define SUM_FILE_SIZE(dbf, delta) \ 47static inline off_t
48 ((dbf)->mapped_off + (dbf)->mapped_size + (delta)) 48SUM_FILE_SIZE (GDBM_FILE dbf, off_t delta)
49{
50 if (delta >= 0
51 && off_t_sum_ok (dbf->mapped_off, dbf->mapped_size)
52 && off_t_sum_ok (dbf->mapped_off + dbf->mapped_size, delta))
53 return dbf->mapped_off + dbf->mapped_size + delta;
54 return -1;
55}
49 56
50/* Store the size of the GDBM file DBF in *PSIZE. 57/* Store the size of the GDBM file DBF in *PSIZE.
51 Return 0 on success and -1 on failure. */ 58 Return 0 on success and -1 on failure. */
@@ -182,6 +189,17 @@ _gdbm_mapped_remap (GDBM_FILE dbf, off_t size, int flag)
182{ 189{
183 off_t file_size, pos; 190 off_t file_size, pos;
184 191
192 if (size < 0)
193 {
194 errno = EINVAL;
195 GDBM_SET_ERRNO (dbf, GDBM_FILE_SEEK_ERROR, TRUE);
196 return -1;
197 }
198
199 if (size < dbf->mapped_size)
200 /* Nothing to do */
201 return 0;
202
185 if (_gdbm_file_size (dbf, &file_size)) 203 if (_gdbm_file_size (dbf, &file_size))
186 { 204 {
187 SAVE_ERRNO (_gdbm_mapped_unmap (dbf)); 205 SAVE_ERRNO (_gdbm_mapped_unmap (dbf));
diff --git a/src/recover.c b/src/recover.c
index d6d4ff9..721c23f 100644
--- a/src/recover.c
+++ b/src/recover.c
@@ -312,15 +312,30 @@ run_recovery (GDBM_FILE dbf, GDBM_FILE new_dbf, gdbm_recovery *rcvr, int flags)
312 312
313 if (gdbm_store (new_dbf, key, data, GDBM_INSERT) != 0) 313 if (gdbm_store (new_dbf, key, data, GDBM_INSERT) != 0)
314 { 314 {
315 if (flags & GDBM_RCVR_ERRFUN) 315 switch (gdbm_last_errno (new_dbf))
316 rcvr->errfun (rcvr->data, 316 {
317 _("fatal: can't store element %d:%d (%lu:%d): %s"), 317 case GDBM_CANNOT_REPLACE:
318 bucket_dir, i, 318 rcvr->duplicate_keys++;
319 (unsigned long) dbf->bucket->h_table[i].data_pointer, 319 if (flags & GDBM_RCVR_ERRFUN)
320 dbf->bucket->h_table[i].key_size 320 rcvr->errfun (rcvr->data,
321 _("ignoring duplicate key %d:%d (%lu:%d)"),
322 bucket_dir, i,
323 (unsigned long) dbf->bucket->h_table[i].data_pointer,
324 dbf->bucket->h_table[i].key_size
325 + dbf->bucket->h_table[i].data_size);
326 break;
327
328 default:
329 if (flags & GDBM_RCVR_ERRFUN)
330 rcvr->errfun (rcvr->data,
331 _("fatal: can't store element %d:%d (%lu:%d): %s"),
332 bucket_dir, i,
333 (unsigned long) dbf->bucket->h_table[i].data_pointer,
334 dbf->bucket->h_table[i].key_size
321 + dbf->bucket->h_table[i].data_size, 335 + dbf->bucket->h_table[i].data_size,
322 gdbm_db_strerror (new_dbf)); 336 gdbm_db_strerror (new_dbf));
323 return -1; 337 return -1;
338 }
324 } 339 }
325 } 340 }
326 } 341 }
@@ -356,6 +371,7 @@ gdbm_recover (GDBM_FILE dbf, gdbm_recovery *rcvr, int flags)
356 rcvr->recovered_buckets = 0; 371 rcvr->recovered_buckets = 0;
357 rcvr->failed_keys = 0; 372 rcvr->failed_keys = 0;
358 rcvr->failed_buckets = 0; 373 rcvr->failed_buckets = 0;
374 rcvr->duplicate_keys = 0;
359 rcvr->backup_name = NULL; 375 rcvr->backup_name = NULL;
360 376
361 rc = 0; 377 rc = 0;
diff --git a/src/systems.h b/src/systems.h
index 66955dd..c678573 100644
--- a/src/systems.h
+++ b/src/systems.h
@@ -33,6 +33,7 @@
33#include <unistd.h> 33#include <unistd.h>
34#include <fcntl.h> 34#include <fcntl.h>
35#include <errno.h> 35#include <errno.h>
36#include <limits.h>
36 37
37#ifndef SEEK_SET 38#ifndef SEEK_SET
38# define SEEK_SET 0 39# define SEEK_SET 0

Return to:

Send suggestions and report system problems to the System administrator.