version 1.57, 2018/08/24 16:01:57 |
version 1.58, 2019/03/11 00:14:44 |
Line 39 __RCSID("$NetBSD$"); |
|
Line 39 __RCSID("$NetBSD$"); |
|
#include "dist/mandoc.h" |
#include "dist/mandoc.h" |
#include "dist/mdoc.h" |
#include "dist/mdoc.h" |
#include "dist/roff.h" |
#include "dist/roff.h" |
|
#include "dist/roff_int.h" |
|
#include "dist/mandoc_parse.h" |
|
|
#define BUFLEN 1024 |
#define BUFLEN 1024 |
#define MDOC 0 //If the page is of mdoc(7) type |
#define MDOC 0 //If the page is of mdoc(7) type |
Line 98 static void init_secbuffs(mandb_rec *); |
|
Line 100 static void init_secbuffs(mandb_rec *); |
|
static void free_secbuffs(mandb_rec *); |
static void free_secbuffs(mandb_rec *); |
static int check_md5(const char *, sqlite3 *, char **, void *, size_t); |
static int check_md5(const char *, sqlite3 *, char **, void *, size_t); |
static void cleanup(mandb_rec *); |
static void cleanup(mandb_rec *); |
static void set_section(const struct roff_man *, mandb_rec *); |
static void set_section(const struct roff_meta *, mandb_rec *); |
static void set_machine(const struct roff_man *, mandb_rec *); |
static void set_machine(const struct roff_meta *, mandb_rec *); |
static int insert_into_db(sqlite3 *, mandb_rec *); |
static int insert_into_db(sqlite3 *, mandb_rec *); |
static void begin_parse(const char *, struct mparse *, mandb_rec *, |
static void begin_parse(const char *, struct mparse *, mandb_rec *, int); |
const void *, size_t len); |
static void proff_node(const struct roff_node *, mandb_rec *, |
static void proff_node(const struct roff_node *, mandb_rec *, struct roff_man *, |
struct roff_meta *, const proff_nf *); |
const proff_nf *); |
|
static void pmdoc_Nm(const struct roff_node *, mandb_rec *); |
static void pmdoc_Nm(const struct roff_node *, mandb_rec *); |
static void pmdoc_Nd(const struct roff_node *, mandb_rec *); |
static void pmdoc_Nd(const struct roff_node *, mandb_rec *); |
static void pmdoc_Sh(const struct roff_node *, mandb_rec *); |
static void pmdoc_Sh(const struct roff_node *, mandb_rec *); |
Line 336 main(int argc, char *argv[]) |
|
Line 337 main(int argc, char *argv[]) |
|
|
|
init_secbuffs(&rec); |
init_secbuffs(&rec); |
mchars_alloc(); |
mchars_alloc(); |
mp = mparse_alloc(0, MANDOCERR_MAX, NULL, MANDOC_OS_OTHER, NULL); |
mp = mparse_alloc(MPARSE_SO | MPARSE_UTF8 | MPARSE_LATIN1 | |
|
MPARSE_VALIDATE, MANDOC_OS_OTHER, NULL); |
|
|
if (manconf) { |
if (manconf) { |
char *arg; |
char *arg; |
Line 579 build_file_cache(sqlite3 *db, const char |
|
Line 581 build_file_cache(sqlite3 *db, const char |
|
sqlite3_finalize(stmt); |
sqlite3_finalize(stmt); |
} |
} |
|
|
static void |
|
update_existing_entry(sqlite3 *db, const char *file, const char *hash, |
|
mandb_rec *rec, int *new_count, int *link_count, int *err_count) |
|
{ |
|
int update_count, rc, idx; |
|
const char *inner_sqlstr; |
|
sqlite3_stmt *inner_stmt; |
|
|
|
update_count = sqlite3_total_changes(db); |
|
inner_sqlstr = "UPDATE mandb_meta SET device = :device," |
|
" inode = :inode, mtime = :mtime WHERE" |
|
" md5_hash = :md5 AND file = :file AND" |
|
" (device <> :device2 OR inode <> " |
|
" :inode2 OR mtime <> :mtime2)"; |
|
rc = sqlite3_prepare_v2(db, inner_sqlstr, -1, &inner_stmt, NULL); |
|
if (rc != SQLITE_OK) { |
|
if (mflags.verbosity) |
|
warnx("%s", sqlite3_errmsg(db)); |
|
return; |
|
} |
|
idx = sqlite3_bind_parameter_index(inner_stmt, ":device"); |
|
sqlite3_bind_int64(inner_stmt, idx, rec->device); |
|
idx = sqlite3_bind_parameter_index(inner_stmt, ":inode"); |
|
sqlite3_bind_int64(inner_stmt, idx, rec->inode); |
|
idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime"); |
|
sqlite3_bind_int64(inner_stmt, idx, rec->mtime); |
|
idx = sqlite3_bind_parameter_index(inner_stmt, ":md5"); |
|
sqlite3_bind_text(inner_stmt, idx, hash, -1, NULL); |
|
idx = sqlite3_bind_parameter_index(inner_stmt, ":file"); |
|
sqlite3_bind_text(inner_stmt, idx, file, -1, NULL); |
|
idx = sqlite3_bind_parameter_index(inner_stmt, ":device2"); |
|
sqlite3_bind_int64(inner_stmt, idx, rec->device); |
|
idx = sqlite3_bind_parameter_index(inner_stmt, ":inode2"); |
|
sqlite3_bind_int64(inner_stmt, idx, rec->inode); |
|
idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime2"); |
|
sqlite3_bind_int64(inner_stmt, idx, rec->mtime); |
|
|
|
rc = sqlite3_step(inner_stmt); |
|
if (rc == SQLITE_DONE) { |
|
/* Check if an update has been performed. */ |
|
if (update_count != sqlite3_total_changes(db)) { |
|
if (mflags.verbosity == 2) |
|
printf("Updated %s\n", file); |
|
(*new_count)++; |
|
} else { |
|
/* Otherwise it was a hardlink. */ |
|
(*link_count)++; |
|
} |
|
} else { |
|
if (mflags.verbosity == 2) |
|
warnx("Could not update the meta data for %s", file); |
|
(*err_count)++; |
|
} |
|
sqlite3_finalize(inner_stmt); |
|
} |
|
|
|
/* read_and_decompress -- |
/* read_and_decompress -- |
* Reads the given file into memory. If it is compressed, decompress |
* Reads the given file into memory. If it is compressed, decompress |
* it before returning to the caller. |
* it before returning to the caller. |
|
|
return -1; |
return -1; |
} |
} |
|
|
|
static void |
|
update_existing_entry(sqlite3 *db, const char *file, const char *hash, |
|
mandb_rec *rec, int *new_count, int *link_count, int *err_count) |
|
{ |
|
int update_count, rc, idx; |
|
const char *inner_sqlstr; |
|
sqlite3_stmt *inner_stmt; |
|
|
|
update_count = sqlite3_total_changes(db); |
|
inner_sqlstr = "UPDATE mandb_meta SET device = :device," |
|
" inode = :inode, mtime = :mtime WHERE" |
|
" md5_hash = :md5 AND file = :file AND" |
|
" (device <> :device2 OR inode <> " |
|
" :inode2 OR mtime <> :mtime2)"; |
|
rc = sqlite3_prepare_v2(db, inner_sqlstr, -1, &inner_stmt, NULL); |
|
if (rc != SQLITE_OK) { |
|
if (mflags.verbosity) |
|
warnx("%s", sqlite3_errmsg(db)); |
|
return; |
|
} |
|
idx = sqlite3_bind_parameter_index(inner_stmt, ":device"); |
|
sqlite3_bind_int64(inner_stmt, idx, rec->device); |
|
idx = sqlite3_bind_parameter_index(inner_stmt, ":inode"); |
|
sqlite3_bind_int64(inner_stmt, idx, rec->inode); |
|
idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime"); |
|
sqlite3_bind_int64(inner_stmt, idx, rec->mtime); |
|
idx = sqlite3_bind_parameter_index(inner_stmt, ":md5"); |
|
sqlite3_bind_text(inner_stmt, idx, hash, -1, NULL); |
|
idx = sqlite3_bind_parameter_index(inner_stmt, ":file"); |
|
sqlite3_bind_text(inner_stmt, idx, file, -1, NULL); |
|
idx = sqlite3_bind_parameter_index(inner_stmt, ":device2"); |
|
sqlite3_bind_int64(inner_stmt, idx, rec->device); |
|
idx = sqlite3_bind_parameter_index(inner_stmt, ":inode2"); |
|
sqlite3_bind_int64(inner_stmt, idx, rec->inode); |
|
idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime2"); |
|
sqlite3_bind_int64(inner_stmt, idx, rec->mtime); |
|
|
|
rc = sqlite3_step(inner_stmt); |
|
if (rc == SQLITE_DONE) { |
|
/* Check if an update has been performed. */ |
|
if (update_count != sqlite3_total_changes(db)) { |
|
if (mflags.verbosity == 2) |
|
printf("Updated %s\n", file); |
|
(*new_count)++; |
|
} else { |
|
/* Otherwise it was a hardlink. */ |
|
(*link_count)++; |
|
} |
|
} else { |
|
if (mflags.verbosity == 2) |
|
warnx("Could not update the meta data for %s", file); |
|
(*err_count)++; |
|
} |
|
sqlite3_finalize(inner_stmt); |
|
} |
|
|
/* update_db -- |
/* update_db -- |
* Does an incremental updation of the database by checking the file_cache. |
* Does an incremental updation of the database by checking the file_cache. |
* It parses and adds the pages which are present in file_cache, |
* It parses and adds the pages which are present in file_cache, |
Line 755 update_db(sqlite3 *db, struct mparse *mp |
|
Line 757 update_db(sqlite3 *db, struct mparse *mp |
|
sqlite3_finalize(stmt); |
sqlite3_finalize(stmt); |
|
|
for ( ; rows != NULL; free(parent), free(file), free(buf)) { |
for ( ; rows != NULL; free(parent), free(file), free(buf)) { |
|
int fd; |
|
|
row = rows; |
row = rows; |
rows = rows->next; |
rows = rows->next; |
|
|
Line 765 update_db(sqlite3 *db, struct mparse *mp |
|
Line 769 update_db(sqlite3 *db, struct mparse *mp |
|
file = row->file; |
file = row->file; |
free(row); |
free(row); |
|
|
|
// XXX: reading twice! |
if (read_and_decompress(file, &buf, &buflen)) { |
if (read_and_decompress(file, &buf, &buflen)) { |
err_count++; |
err_count++; |
continue; |
continue; |
} |
} |
|
if ((fd = mparse_open(mp, file)) == -1) { |
|
err_count++; |
|
continue; |
|
} |
|
|
md5_status = check_md5(file, db, &md5sum, buf, buflen); |
md5_status = check_md5(file, db, &md5sum, buf, buflen); |
assert(md5sum != NULL); |
assert(md5sum != NULL); |
if (md5_status == -1) { |
if (md5_status == -1) { |
Line 776 update_db(sqlite3 *db, struct mparse *mp |
|
Line 786 update_db(sqlite3 *db, struct mparse *mp |
|
warnx("An error occurred in checking md5 value" |
warnx("An error occurred in checking md5 value" |
" for file %s", file); |
" for file %s", file); |
err_count++; |
err_count++; |
|
close(fd); |
continue; |
continue; |
} |
} |
|
|
Line 787 update_db(sqlite3 *db, struct mparse *mp |
|
Line 798 update_db(sqlite3 *db, struct mparse *mp |
|
update_existing_entry(db, file, md5sum, rec, |
update_existing_entry(db, file, md5sum, rec, |
&new_count, &link_count, &err_count); |
&new_count, &link_count, &err_count); |
free(md5sum); |
free(md5sum); |
|
close(fd); |
continue; |
continue; |
} |
} |
|
|
Line 802 update_db(sqlite3 *db, struct mparse *mp |
|
Line 814 update_db(sqlite3 *db, struct mparse *mp |
|
"not index `%s'", parent, file); |
"not index `%s'", parent, file); |
err_count++; |
err_count++; |
free(md5sum); |
free(md5sum); |
|
close(fd); |
continue; |
continue; |
} |
} |
|
|
Line 810 update_db(sqlite3 *db, struct mparse *mp |
|
Line 823 update_db(sqlite3 *db, struct mparse *mp |
|
rec->md5_hash = md5sum; |
rec->md5_hash = md5sum; |
rec->file_path = estrdup(file); |
rec->file_path = estrdup(file); |
// file_path is freed by insert_into_db itself. |
// file_path is freed by insert_into_db itself. |
begin_parse(file, mp, rec, buf, buflen); |
begin_parse(file, mp, rec, fd); |
if (insert_into_db(db, rec) < 0) { |
if (insert_into_db(db, rec) < 0) { |
if (mflags.verbosity) |
if (mflags.verbosity) |
warnx("Error in indexing `%s'", file); |
warnx("Error in indexing `%s'", file); |
Line 819 update_db(sqlite3 *db, struct mparse *mp |
|
Line 832 update_db(sqlite3 *db, struct mparse *mp |
|
new_count++; |
new_count++; |
} |
} |
} |
} |
|
close(fd); |
} |
} |
|
|
if (mflags.verbosity == 2) { |
if (mflags.verbosity == 2) { |
Line 859 update_db(sqlite3 *db, struct mparse *mp |
|
Line 873 update_db(sqlite3 *db, struct mparse *mp |
|
* parses the man page using libmandoc |
* parses the man page using libmandoc |
*/ |
*/ |
static void |
static void |
begin_parse(const char *file, struct mparse *mp, mandb_rec *rec, |
begin_parse(const char *file, struct mparse *mp, mandb_rec *rec, int fd) |
const void *buf, size_t len) |
|
{ |
{ |
struct roff_man *roff; |
struct roff_meta *roff; |
mparse_reset(mp); |
mparse_reset(mp); |
|
|
rec->xr_found = 0; |
rec->xr_found = 0; |
|
|
if (mparse_readmem(mp, buf, len, file) >= MANDOCLEVEL_BADARG) { |
mparse_readfd(mp, fd, file); |
/* Printing this warning at verbosity level 2 |
roff = mparse_result(mp); |
* because some packages from pkgsrc might trigger several |
|
* of such warnings. |
|
*/ |
|
if (mflags.verbosity == 2) |
|
warnx("%s: Parse failure", file); |
|
return; |
|
} |
|
|
|
mparse_result(mp, &roff, NULL); |
|
if (roff == NULL) { |
if (roff == NULL) { |
if (mflags.verbosity == 2) |
if (mflags.verbosity == 2) |
warnx("Not a roff(7) page"); |
warnx("Not a roff(7) page"); |
Line 885 begin_parse(const char *file, struct mpa |
|
Line 889 begin_parse(const char *file, struct mpa |
|
} |
} |
|
|
if (roff->macroset == MACROSET_MDOC) { |
if (roff->macroset == MACROSET_MDOC) { |
mdoc_validate(roff); |
|
rec->page_type = MDOC; |
rec->page_type = MDOC; |
proff_node(roff->first->child, rec, roff, mdocs); |
proff_node(roff->first->child, rec, roff, mdocs); |
} else if (roff->macroset == MACROSET_MAN) { |
} else if (roff->macroset == MACROSET_MAN) { |
man_validate(roff); |
|
rec->page_type = MAN; |
rec->page_type = MAN; |
proff_node(roff->first->child, rec, roff, mans); |
proff_node(roff->first->child, rec, roff, mans); |
} else |
} else |
Line 904 begin_parse(const char *file, struct mpa |
|
Line 906 begin_parse(const char *file, struct mpa |
|
* (Which should be the first character of the string). |
* (Which should be the first character of the string). |
*/ |
*/ |
static void |
static void |
set_section(const struct roff_man *rm, mandb_rec *rec) |
set_section(const struct roff_meta *rm, mandb_rec *rec) |
{ |
{ |
if (!rm) |
if (!rm) |
return; |
return; |
const char *s = rm->meta.msec == NULL ? "?" : rm->meta.msec; |
const char *s = rm->msec == NULL ? "?" : rm->msec; |
easprintf(&rec->section, "%s", s); |
easprintf(&rec->section, "%s", s); |
if (rec->section[0] == '?' && mflags.verbosity == 2) |
if (rec->section[0] == '?' && mflags.verbosity == 2) |
warnx("%s: Missing section number", rec->file_path); |
warnx("%s: Missing section number", rec->file_path); |
Line 919 set_section(const struct roff_man *rm, m |
|
Line 921 set_section(const struct roff_man *rm, m |
|
* Extracts the machine architecture information if available. |
* Extracts the machine architecture information if available. |
*/ |
*/ |
static void |
static void |
set_machine(const struct roff_man *rm, mandb_rec *rec) |
set_machine(const struct roff_meta *rm, mandb_rec *rec) |
{ |
{ |
if (rm == NULL) |
if (rm == NULL) |
return; |
return; |
if (rm->meta.arch) |
if (rm->arch) |
rec->machine = estrdup(rm->meta.arch); |
rec->machine = estrdup(rm->arch); |
} |
} |
|
|
/* |
/* |
Line 1147 mdoc_parse_section(enum roff_sec sec, co |
|
Line 1149 mdoc_parse_section(enum roff_sec sec, co |
|
|
|
static void |
static void |
proff_node(const struct roff_node * n, mandb_rec * rec, |
proff_node(const struct roff_node * n, mandb_rec * rec, |
struct roff_man * roff, const proff_nf * func) |
struct roff_meta * roff, const proff_nf * func) |
{ |
{ |
if (n == NULL) |
if (n == NULL) |
return; |
return; |