diff --git a/.gitignore b/.gitignore index de66884..2693d7e 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ config.h *.a *~ pkg +pkg2sqfs diff --git a/Makefile.am b/Makefile.am index 1afb94f..99f3a2e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -10,3 +10,4 @@ EXTRA_DIST = autogen.sh LICENSE README.md include lib/Makemodule.am include main/Makemodule.am +include sqfs/Makemodule.am diff --git a/configure.ac b/configure.ac index 5377012..51feb93 100644 --- a/configure.ac +++ b/configure.ac @@ -50,6 +50,10 @@ PKG_CHECK_MODULES(XZ, [liblzma >= 5.0.0], [have_lzma="yes"], []) AM_CONDITIONAL([WITH_ZLIB], [test "x$have_zlib" == "xyes"]) AM_CONDITIONAL([WITH_LZMA], [test "x$have_lzma" == "xyes"]) +AM_CONDITIONAL([HAVE_LIBBSD], [true]) +PKG_CHECK_MODULES(LIBBSD, [libbsd], [], + [AC_MSG_ERROR([missing libbsd])]) + ##### generate output ##### AC_CONFIG_HEADERS([config.h]) diff --git a/sqfs/Makemodule.am b/sqfs/Makemodule.am new file mode 100644 index 0000000..881fabc --- /dev/null +++ b/sqfs/Makemodule.am @@ -0,0 +1,22 @@ +pkg2sqfs_SOURCES = sqfs/squashfs.h sqfs/super.c sqfs/pkg2sqfs.c +pkg2sqfs_SOURCES += sqfs/vfs.c sqfs/pkg2sqfs.h sqfs/block.c +pkg2sqfs_SOURCES += sqfs/meta.c sqfs/meta_writer.c sqfs/table.c +pkg2sqfs_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/sqfs +pkg2sqfs_CFLAGS = $(AM_CFLAGS) +pkg2sqfs_LDADD = libpkg.a libutil.a libfilelist.a libcomp.a + +if WITH_LZMA +pkg2sqfs_LDADD += $(XZ_LIBS) +endif + +if WITH_ZLIB +pkg2sqfs_LDADD += $(ZLIB_LIBS) +endif + +if HAVE_LIBBSD +pkg2sqfs_CPPFLAGS += -DHAVE_LIBBSD +pkg2sqfs_CFLAGS += $(LIBBSD_CFLAGS) +pkg2sqfs_LDADD += $(LIBBSD_LIBS) +endif + +bin_PROGRAMS += pkg2sqfs diff --git a/sqfs/block.c b/sqfs/block.c new file mode 100644 index 0000000..c97a36a --- /dev/null +++ b/sqfs/block.c @@ -0,0 +1,212 @@ +/* SPDX-License-Identifier: ISC */ +#include "pkg2sqfs.h" + +static int write_block(node_t *node, sqfs_info_t *info) +{ + size_t idx, bs; + ssize_t ret; + + /* TODO: try to compress the block */ + + idx = info->file_block_count++; + bs = info->super.block_size; + + ret = write_retry(info->outfd, info->block, bs); + if (ret < 0) { + perror("writing to output file"); + return -1; + } + + if ((size_t)ret < bs) { + fputs("write to output file truncated\n", stderr); + return -1; + } + + node->data.file->blocksizes[idx] = bs | (1 << 24); + + info->super.flags |= SQFS_FLAG_UNCOMPRESSED_DATA; + info->super.bytes_used += bs; + return 0; +} + +static int flush_fragments(sqfs_info_t *info) +{ + size_t newsz, size; + file_info_t *fi; + uint64_t offset; + ssize_t ret; + void *new; + + if (info->num_fragments == info->max_fragments) { + newsz = info->max_fragments ? info->max_fragments * 2 : 16; + new = realloc(info->fragments, + sizeof(info->fragments[0]) * newsz); + + if (new == NULL) { + perror("appending to fragment table"); + return -1; + } + + info->max_fragments = newsz; + info->fragments = new; + } + + offset = info->super.bytes_used; + size = info->frag_offset; + + for (fi = info->frag_list; fi != NULL; fi = fi->frag_next) + fi->fragment = info->num_fragments; + + info->fragments[info->num_fragments].start_offset = htole64(offset); + info->fragments[info->num_fragments].size = htole32(size | (1 << 24)); + info->fragments[info->num_fragments].pad0 = 0; + + info->num_fragments += 1; + + /* TODO: try to compress the fragments */ + + ret = write_retry(info->outfd, info->fragment, size); + if (ret < 0) { + perror("writing to output file"); + return -1; + } + + if ((size_t)ret < size) { + fputs("write to output file truncated\n", stderr); + return -1; + } + + memset(info->fragment, 0, info->super.block_size); + + info->super.bytes_used += size; + info->frag_offset = 0; + info->frag_list = NULL; + + info->super.flags &= ~SQFS_FLAG_NO_FRAGMENTS; + info->super.flags |= SQFS_FLAG_UNCOMPRESSED_FRAGMENTS; + info->super.flags |= SQFS_FLAG_ALWAYS_FRAGMENTS; + return 0; +} + +static int add_fragment(file_info_t *fi, sqfs_info_t *info, size_t size) +{ + if (info->frag_offset + size > info->super.block_size) { + if (flush_fragments(info)) + return -1; + } + + fi->fragment_offset = info->frag_offset; + fi->frag_next = info->frag_list; + info->frag_list = fi; + + memcpy((char *)info->fragment + info->frag_offset, info->block, size); + info->frag_offset += size; + return 0; +} + +static int process_file(node_t *node, sqfs_info_t *info) +{ + uint64_t count = node->data.file->size; + int ret; + size_t diff; + + node->data.file->startblock = info->super.bytes_used; + info->file_block_count = 0; + + while (count != 0) { + diff = count > (uint64_t)info->super.block_size ? + info->super.block_size : count; + + ret = pkg_reader_read_payload(info->rd, info->block, diff); + if (ret < 0) + return -1; + if ((size_t)ret < diff) + goto fail_trunc; + + if (diff < info->super.block_size) { + if (add_fragment(node->data.file, info, diff)) + return -1; + } else { + if (write_block(node, info)) + return -1; + } + + count -= diff; + } + + return 0; +fail_trunc: + fprintf(stderr, "%s: truncated data block\n", + pkg_reader_get_filename(info->rd)); + return -1; +} + +int pkg_data_to_sqfs(sqfs_info_t *info) +{ + file_data_t frec; + record_t *hdr; + node_t *node; + int ret; + + if (pkg_reader_rewind(info->rd)) + return -1; + + memset(info->fragment, 0, info->super.block_size); + + for (;;) { + ret = pkg_reader_get_next_record(info->rd); + if (ret == 0) + break; + if (ret < 0) + return -1; + + hdr = pkg_reader_current_record_header(info->rd); + if (hdr->magic != PKG_MAGIC_DATA) + continue; + + for (;;) { + ret = pkg_reader_read_payload(info->rd, &frec, + sizeof(frec)); + if (ret == 0) + break; + if (ret < 0) + return -1; + if ((size_t)ret < sizeof(frec)) + goto fail_trunc; + + frec.id = le32toh(frec.id); + + node = vfs_node_from_file_id(&info->fs, frec.id); + if (node == NULL) + goto fail_meta; + + if (process_file(node, info)) + return -1; + } + } + + if (info->frag_offset != 0) { + if (flush_fragments(info)) + return -1; + } + + return 0; +fail_meta: + fprintf(stderr, "%s: missing meta information for file %u\n", + pkg_reader_get_filename(info->rd), (unsigned int)frec.id); + return -1; +fail_trunc: + fprintf(stderr, "%s: truncated file data record\n", + pkg_reader_get_filename(info->rd)); + return -1; +} + +int sqfs_write_fragment_table(sqfs_info_t *info) +{ + info->super.fragment_entry_count = info->num_fragments; + + return sqfs_write_table(info, info->fragments, + sizeof(info->fragments[0]), + info->num_fragments, + &info->super.fragment_table_start); +} diff --git a/sqfs/meta.c b/sqfs/meta.c new file mode 100644 index 0000000..0ff6217 --- /dev/null +++ b/sqfs/meta.c @@ -0,0 +1,305 @@ +/* SPDX-License-Identifier: ISC */ +#include "pkg2sqfs.h" + +static size_t hard_link_count(node_t *node) +{ + size_t count; + node_t *n; + + if (S_ISDIR(node->mode)) { + count = 2; + + for (n = node->data.dir->children; n != NULL; n = n->next) + ++count; + + return count; + } + + return 1; +} + +static int write_dir(meta_writer_t *dm, node_t *node) +{ + sqfs_dir_header_t hdr; + sqfs_dir_entry_t ent; + size_t i, count; + node_t *c, *d; + + c = node->data.dir->children; + node->data.dir->size = 0; + + node->data.dir->dref = (dm->written << 16) | dm->offset; + + while (c != NULL) { + count = 0; + + for (d = c; d != NULL; d = d->next) { + if ((d->inode_ref >> 16) != (c->inode_ref >> 16)) + break; + if ((d->inode_num - c->inode_num) > 0xFFFF) + break; + count += 1; + } + + hdr.count = htole32(count - 1); + hdr.start_block = htole32(c->inode_ref >> 16); + hdr.inode_number = htole32(c->inode_num); + + if (meta_writer_append(dm, &hdr, sizeof(hdr))) + return -1; + + d = c; + + for (i = 0; i < count; ++i) { + ent.offset = htole16(c->inode_ref & 0x0000FFFF); + ent.inode_number = htole16(c->inode_num - d->inode_num); + ent.type = htole16(c->type); + ent.size = htole16(strlen(c->name) - 1); + + if (meta_writer_append(dm, &ent, sizeof(ent))) + return -1; + if (meta_writer_append(dm, c->name, strlen(c->name))) + return -1; + + c = c->next; + } + + node->data.dir->size += sizeof(hdr) + count * sizeof(ent); + } + return 0; +} + +static int write_inode(sqfs_info_t *info, meta_writer_t *im, meta_writer_t *dm, + node_t *node) +{ + file_info_t *fi = NULL; + sqfs_inode_t base; + uint32_t bs; + uint64_t i; + + node->inode_ref = (im->written << 16) | im->offset; + + base.mode = htole16(node->mode); + base.uid_idx = htole16(node->uid_idx); + base.gid_idx = htole16(node->gid_idx); + base.mod_time = htole32(info->super.modification_time); + base.inode_number = htole32(node->inode_num); + + switch (node->mode & S_IFMT) { + case S_IFLNK: node->type = SQFS_INODE_SLINK; break; + case S_IFBLK: node->type = SQFS_INODE_BDEV; break; + case S_IFCHR: node->type = SQFS_INODE_CDEV; break; + case S_IFDIR: + if (write_dir(dm, node)) + return -1; + + node->type = SQFS_INODE_DIR; + i = node->data.dir->dref; + + if ((i >> 16) > 0xFFFFFFFFUL || node->data.dir->size > 0xFFFF) + node->type = SQFS_INODE_EXT_DIR; + break; + case S_IFREG: + fi = node->data.file; + node->type = SQFS_INODE_FILE; + + if (fi->startblock > 0xFFFFFFFFUL || fi->size > 0xFFFFFFFFUL || + hard_link_count(node) > 1) { + node->type = SQFS_INODE_EXT_FILE; + } + break; + default: + assert(0); + } + + base.type = htole16(node->type); + + if (meta_writer_append(im, &base, sizeof(base))) + return -1; + + switch (node->type) { + case SQFS_INODE_SLINK: { + sqfs_inode_slink_t slink = { + .nlink = htole32(hard_link_count(node)), + .target_size = htole32(strlen(node->data.symlink)), + }; + + if (meta_writer_append(im, &slink, sizeof(slink))) + return -1; + if (meta_writer_append(im, node->data.symlink, + le32toh(slink.target_size))) { + return -1; + } + break; + } + case SQFS_INODE_BDEV: + case SQFS_INODE_CDEV: { + sqfs_inode_dev_t dev = { + .nlink = htole32(hard_link_count(node)), + .devno = htole32(node->data.device), + }; + + return meta_writer_append(im, &dev, sizeof(dev)); + } + case SQFS_INODE_EXT_FILE: { + sqfs_inode_file_ext_t ext = { + .blocks_start = htole64(fi->startblock), + .file_size = htole64(fi->size), + .sparse = htole64(0xFFFFFFFFFFFFFFFFUL), + .nlink = htole32(hard_link_count(node)), + .fragment_idx = htole32(fi->fragment), + .fragment_offset = htole32(fi->fragment_offset), + .xattr_idx = htole32(0xFFFFFFFF), + }; + + if (meta_writer_append(im, &ext, sizeof(ext))) + return -1; + goto out_file_blocks; + } + case SQFS_INODE_FILE: { + sqfs_inode_file_t reg = { + .blocks_start = htole32(fi->startblock), + .fragment_index = htole32(fi->fragment), + .fragment_offset = htole32(fi->fragment_offset), + .file_size = htole32(fi->size), + }; + + if (meta_writer_append(im, ®, sizeof(reg))) + return -1; + goto out_file_blocks; + } + case SQFS_INODE_DIR: { + sqfs_inode_dir_t dir = { + .start_block = htole32(node->data.dir->dref >> 16), + .nlink = htole32(hard_link_count(node)), + .size = htole16(node->data.dir->size), + .offset = htole16(node->data.dir->dref & 0xFFFF), + .parent_inode = node->parent ? + htole32(node->parent->inode_num) : htole32(1), + }; + + return meta_writer_append(im, &dir, sizeof(dir)); + } + case SQFS_INODE_EXT_DIR: { + sqfs_inode_dir_ext_t ext = { + .nlink = htole32(hard_link_count(node)), + .size = htole32(node->data.dir->size), + .start_block = htole32(node->data.dir->dref >> 16), + .parent_inode = node->parent ? + htole32(node->parent->inode_num) : htole32(1), + .inodex_count = htole32(0), + .offset = htole16(node->data.dir->dref & 0xFFFF), + .xattr_idx = htole32(0xFFFFFFFF), + }; + + return meta_writer_append(im, &ext, sizeof(ext)); + } + default: + assert(0); + } + return 0; +out_file_blocks: + for (i = 0; i < fi->size / info->super.block_size; ++i) { + bs = htole32(fi->blocksizes[i]); + + if (meta_writer_append(im, &bs, sizeof(bs))) + return -1; + } + return 0; +} + +int sqfs_write_inodes(sqfs_info_t *info) +{ + meta_writer_t *im, *dm; + size_t i, diff; + ssize_t ret; + FILE *tmp; + int tmpfd; + + tmp = tmpfile(); + if (tmp == NULL) { + perror("tmpfile"); + return -1; + } + + tmpfd = fileno(tmp); + + im = meta_writer_create(info->outfd); + if (im == NULL) + goto fail_tmp; + + dm = meta_writer_create(tmpfd); + if (dm == NULL) + goto fail_im; + + for (i = 2; i < info->fs.num_inodes; ++i) { + if (write_inode(info, im, dm, info->fs.node_tbl[i])) + goto fail; + } + + if (meta_writer_flush(im)) + goto fail; + + if (meta_writer_flush(dm)) + goto fail; + + info->super.root_inode_ref = info->fs.root->inode_ref; + + info->super.inode_table_start = info->super.bytes_used; + info->super.bytes_used += im->written; + + info->super.directory_table_start = info->super.bytes_used; + info->super.bytes_used += dm->written; + + if (lseek(tmpfd, 0, SEEK_SET) == (off_t)-1) { + perror("rewind on directory temp file"); + goto fail; + } + + for (;;) { + ret = read_retry(tmpfd, dm->data, sizeof(dm->data)); + + if (ret < 0) { + perror("read from temp file"); + goto fail; + } + if (ret == 0) + break; + + diff = ret; + ret = write_retry(info->outfd, dm->data, diff); + + if (ret < 0) { + perror("write to image file"); + goto fail; + } + if ((size_t)ret < diff) { + fputs("copying meta data to image file: " + "truncated write\n", stderr); + goto fail; + } + } + + info->super.flags |= SQFS_FLAG_UNCOMPRESSED_INODES; + + meta_writer_destroy(dm); + meta_writer_destroy(im); + fclose(tmp); + return 0; +fail: + meta_writer_destroy(dm); +fail_im: + meta_writer_destroy(im); +fail_tmp: + fclose(tmp); + return -1; +} + +int sqfs_write_ids(sqfs_info_t *info) +{ + info->super.flags |= SQFS_FLAG_UNCOMPRESSED_IDS; + + return sqfs_write_table(info, info->fs.id_tbl, + sizeof(info->fs.id_tbl[0]), + info->fs.num_ids, &info->super.id_table_start); +} diff --git a/sqfs/meta_writer.c b/sqfs/meta_writer.c new file mode 100644 index 0000000..6df08bd --- /dev/null +++ b/sqfs/meta_writer.c @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: ISC */ +#include "pkg2sqfs.h" + +meta_writer_t *meta_writer_create(int fd) +{ + meta_writer_t *m = calloc(1, sizeof(*m)); + + if (m == NULL) { + perror("creating meta data writer"); + return NULL; + } + + m->outfd = fd; + return m; +} + +void meta_writer_destroy(meta_writer_t *m) +{ + free(m); +} + +int meta_writer_flush(meta_writer_t *m) +{ + ssize_t ret, count; + + /* TODO: compress buffer */ + + if (m->offset == 0) + return 0; + + ((uint16_t *)m->data)[0] = htole16(m->offset | 0x8000); + count = m->offset + 2; + + ret = write_retry(m->outfd, m->data, count); + if (ret < 0) { + perror("writing meta data"); + return -1; + } + + if (ret < count) { + fputs("meta data was truncated\n", stderr); + return -1; + } + + memset(m->data, 0, sizeof(m->data)); + m->offset = 0; + m->written += count; + return 0; +} + +int meta_writer_append(meta_writer_t *m, const void *data, size_t size) +{ + size_t diff; + + while (size != 0) { + diff = sizeof(m->data) - 2 - m->offset; + + if (diff == 0) { + if (meta_writer_flush(m)) + return -1; + diff = sizeof(m->data) - 2; + } + + if (diff > size) + diff = size; + + memcpy(m->data + 2 + m->offset, data, diff); + m->offset += diff; + size -= diff; + data = (const char *)data + diff; + } + + return 0; +} diff --git a/sqfs/pkg2sqfs.c b/sqfs/pkg2sqfs.c new file mode 100644 index 0000000..927b886 --- /dev/null +++ b/sqfs/pkg2sqfs.c @@ -0,0 +1,191 @@ +/* SPDX-License-Identifier: ISC */ +#include "pkg2sqfs.h" + +static struct option long_opts[] = { + { "block-size", required_argument, NULL, 'b' }, + { "force", no_argument, NULL, 'f' }, + { "version", no_argument, NULL, 'V' }, + { "help", no_argument, NULL, 'h' }, +}; + +static const char *short_opts = "b:fhV"; + +extern char *__progname; + +static const char *version_string = +"%s (Pygos %s) %s\n" +"Copyright (c) 2019 David Oberhollenzer\n" +"License ISC \n" +"This is free software: you are free to change and redistribute it.\n" +"There is NO WARRANTY, to the extent permitted by law.\n" +"\n" +"Written by David Oberhollenzer.\n"; + +static const char *help_string = +"Usage: %s [OPTIONS] \n" +"\n" +"Convert a package file into a squashfs image.\n" +"\n" +"Possible options:\n" +"\n" +" --block-size, -b Block size to use for Squashfs image.\n" +" Defaults to %u.\n" +" --force, -f Overwrite the output file if it already exists.\n" +" --help, -h Print help text and exit.\n" +" --version, -V Print version information and exit.\n" +"\n"; + +static void print_tree(int level, node_t *n) +{ + int i; + + for (i = 0; i < (level - 1); ++i) + fputs("| ", stdout); + + switch (n->mode & S_IFMT) { + case S_IFDIR: + printf("%s%s/ (%u, %o)\n", level ? "+- " : "", n->name, + n->inode_num, (unsigned int)n->mode); + + n = n->data.dir->children; + ++level; + + while (n != NULL) { + print_tree(level, n); + n = n->next; + } + break; + case S_IFLNK: + printf("+- %s (%u) -> %s\n", n->name, n->inode_num, + n->data.symlink); + break; + case S_IFBLK: + printf("+- %s (%u) b %lu\n", n->name, n->inode_num, + n->data.device); + break; + case S_IFCHR: + printf("+- %s (%u) c %lu\n", n->name, n->inode_num, + n->data.device); + break; + default: + printf("+- %s (%u)\n", n->name, n->inode_num); + break; + } +} + +int main(int argc, char **argv) +{ + uint32_t blocksize = SQFS_DEFAULT_BLOCK_SIZE, timestamp = 0; + int i, outmode = O_WRONLY | O_CREAT | O_EXCL; + const char *infile, *outfile, *errstr; + int status = EXIT_FAILURE; + sqfs_info_t info; + + memset(&info, 0, sizeof(info)); + + for (;;) { + i = getopt_long(argc, argv, short_opts, long_opts, NULL); + if (i == -1) + break; + + switch (i) { + case 'b': + blocksize = strtonum(optarg, 1024, 0xFFFFFFFF, &errstr); + if (errstr != NULL) { + fprintf(stderr, "Block size '%s': %s\n", + optarg, errstr); + return EXIT_FAILURE; + } + break; + case 'f': + outmode = O_WRONLY | O_CREAT | O_TRUNC; + break; + case 'h': + printf(help_string, __progname, + SQFS_DEFAULT_BLOCK_SIZE); + exit(EXIT_SUCCESS); + case 'V': + printf(version_string, __progname, + PACKAGE_NAME, PACKAGE_VERSION); + exit(EXIT_SUCCESS); + default: + goto fail_arg; + } + } + + if ((optind + 1) >= argc) { + fputs("Missing arguments: input and output files.\n", stderr); + goto fail_arg; + } + + infile = argv[optind++]; + outfile = argv[optind++]; + + info.rd = pkg_reader_open(infile); + if (info.rd == NULL) + return EXIT_FAILURE; + + info.outfd = open(outfile, outmode, 0644); + if (info.outfd < 0) { + perror(outfile); + goto out_pkg_close; + } + + if (sqfs_super_init(&info.super, timestamp, blocksize, SQFS_COMP_GZIP)) + goto out_close; + + info.block = malloc(info.super.block_size * 3); + if (info.block == NULL) { + perror("malloc"); + goto out_close; + } + + info.scratch = (char *)info.block + info.super.block_size; + info.fragment = (char *)info.block + 2 * info.super.block_size; + + info.fs.default_uid = 0; + info.fs.default_gid = 0; + info.fs.default_mode = 0755; + + if (create_vfs_tree(&info)) + goto out_buffer; + + print_tree(0, info.fs.root); + + if (sqfs_super_write(&info)) + goto out_tree; + + if (pkg_data_to_sqfs(&info)) + goto out_fragments; + + free(info.block); + info.block = NULL; + + if (sqfs_write_inodes(&info)) + goto out_fragments; + + if (sqfs_write_fragment_table(&info)) + goto out_fragments; + + if (sqfs_write_ids(&info)) + goto out_fragments; + + if (sqfs_super_write(&info)) + goto out_fragments; + + status = EXIT_SUCCESS; +out_fragments: + free(info.fragments); +out_tree: + destroy_vfs_tree(&info.fs); +out_buffer: + free(info.block); +out_close: + close(info.outfd); +out_pkg_close: + pkg_reader_close(info.rd); + return status; +fail_arg: + fprintf(stderr, "Try `%s --help' for more information.\n", __progname); + return EXIT_FAILURE; +} diff --git a/sqfs/pkg2sqfs.h b/sqfs/pkg2sqfs.h new file mode 100644 index 0000000..9a7dfe1 --- /dev/null +++ b/sqfs/pkg2sqfs.h @@ -0,0 +1,137 @@ +/* SPDX-License-Identifier: ISC */ +#ifndef PKG2SQFS_H +#define PKG2SQFS_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_LIBBSD +#include +#endif + +#include "util/util.h" + +#include "filelist/image_entry.h" +#include "pkg/pkgreader.h" +#include "pkg/pkgio.h" +#include "squashfs.h" + +#include "config.h" + +typedef struct { + uint8_t data[SQFS_META_BLOCK_SIZE + 2]; + size_t offset; + size_t written; + int outfd; +} meta_writer_t; + +typedef struct file_info_t { + struct file_info_t *frag_next; + uint64_t size; + uint64_t startblock; + uint32_t id; + uint32_t fragment; + uint32_t fragment_offset; + uint32_t blocksizes[]; +} file_info_t; + +typedef struct { + struct node_t *children; + uint64_t dref; + size_t size; +} dir_info_t; + +typedef struct node_t { + struct node_t *parent; + struct node_t *next; + const char *name; + + union { + dir_info_t *dir; + const char *symlink; + dev_t device; + file_info_t *file; + } data; + + uint64_t inode_ref; + uint32_t inode_num; + uint16_t type; + uint16_t mode; + uint16_t uid_idx; + uint16_t gid_idx; + + uint8_t extra[]; +} node_t; + +typedef struct { + uint32_t default_uid; + uint32_t default_gid; + uint32_t default_mode; + + node_t **node_tbl; + size_t num_inodes; + + uint32_t *id_tbl; + size_t num_ids; + size_t max_ids; + + node_t *root; +} vfs_t; + +typedef struct { + int outfd; + sqfs_super_t super; + vfs_t fs; + pkg_reader_t *rd; + void *block; + void *scratch; + void *fragment; + + sqfs_fragment_t *fragments; + size_t num_fragments; + size_t max_fragments; + + int file_block_count; + file_info_t *frag_list; + size_t frag_offset; +} sqfs_info_t; + +int pkg_data_to_sqfs(sqfs_info_t *info); + +int sqfs_super_init(sqfs_super_t *s, int64_t timestamp, + uint32_t blocksize, E_SQFS_COMPRESSOR comp); + +int sqfs_super_write(sqfs_info_t *info); + +int create_vfs_tree(sqfs_info_t *info); + +void destroy_vfs_tree(vfs_t *fs); + +node_t *vfs_node_from_file_id(vfs_t *fs, uint32_t id); + +int sqfs_write_inodes(sqfs_info_t *info); + +int sqfs_write_ids(sqfs_info_t *info); + +int sqfs_write_fragment_table(sqfs_info_t *info); + +meta_writer_t *meta_writer_create(int fd); + +void meta_writer_destroy(meta_writer_t *m); + +int meta_writer_flush(meta_writer_t *m); + +int meta_writer_append(meta_writer_t *m, const void *data, size_t size); + +int sqfs_write_table(sqfs_info_t *info, const void *data, size_t entsize, + size_t count, uint64_t *start); + +#endif /* PKG2SQFS_H */ diff --git a/sqfs/squashfs.h b/sqfs/squashfs.h new file mode 100644 index 0000000..adda313 --- /dev/null +++ b/sqfs/squashfs.h @@ -0,0 +1,158 @@ +/* SPDX-License-Identifier: ISC */ +#ifndef SQUASHFS_H +#define SQUASHFS_H + +#include + +#define SQFS_MAGIC 0x73717368 +#define SQFS_VERSION_MAJOR 4 +#define SQFS_VERSION_MINOR 0 +#define SQFS_META_BLOCK_SIZE 8192 +#define SQFS_DEFAULT_BLOCK_SIZE 131072 + +typedef struct { + uint32_t magic; + uint32_t inode_count; + uint32_t modification_time; + uint32_t block_size; + uint32_t fragment_entry_count; + uint16_t compression_id; + uint16_t block_log; + uint16_t flags; + uint16_t id_count; + uint16_t version_major; + uint16_t version_minor; + uint64_t root_inode_ref; + uint64_t bytes_used; + uint64_t id_table_start; + uint64_t xattr_id_table_start; + uint64_t inode_table_start; + uint64_t directory_table_start; + uint64_t fragment_table_start; + uint64_t export_table_start; +} __attribute__((packed)) sqfs_super_t; + +typedef struct { + uint64_t start_offset; + uint32_t size; + uint32_t pad0; +} sqfs_fragment_t; + +typedef struct { + uint16_t type; + uint16_t mode; + uint16_t uid_idx; + uint16_t gid_idx; + uint32_t mod_time; + uint32_t inode_number; +} sqfs_inode_t; + +typedef struct { + uint32_t nlink; + uint32_t devno; +} sqfs_inode_dev_t; + +typedef struct { + uint32_t nlink; + uint32_t devno; + uint32_t xattr_idx; +} sqfs_inode_dev_ext_t; + +typedef struct { + uint32_t nlink; + uint32_t target_size; + uint8_t target[]; +} sqfs_inode_slink_t; + +typedef struct { + uint32_t blocks_start; + uint32_t fragment_index; + uint32_t fragment_offset; + uint32_t file_size; + uint32_t block_sizes[]; +} sqfs_inode_file_t; + +typedef struct { + uint64_t blocks_start; + uint64_t file_size; + uint64_t sparse; + uint32_t nlink; + uint32_t fragment_idx; + uint32_t fragment_offset; + uint32_t xattr_idx; + uint32_t block_sizes[]; +} sqfs_inode_file_ext_t; + +typedef struct { + uint32_t start_block; + uint32_t nlink; + uint16_t size; + uint16_t offset; + uint32_t parent_inode; +} sqfs_inode_dir_t; + +typedef struct { + uint32_t nlink; + uint32_t size; + uint32_t start_block; + uint32_t parent_inode; + uint16_t inodex_count; + uint16_t offset; + uint32_t xattr_idx; +} sqfs_inode_dir_ext_t; + +typedef struct { + uint32_t count; + uint32_t start_block; + uint32_t inode_number; +} sqfs_dir_header_t; + +typedef struct { + uint16_t offset; + uint16_t inode_number; + uint16_t type; + uint16_t size; + uint8_t name[]; +} sqfs_dir_entry_t; + +typedef enum { + SQFS_COMP_GZIP = 1, + SQFS_COMP_LZMA = 2, + SQFS_COMP_LZO = 3, + SQFS_COMP_XZ = 4, + SQFS_COMP_LZ4 = 5, + SQFS_COMP_ZSTD = 6, +} E_SQFS_COMPRESSOR; + +typedef enum { + SQFS_FLAG_UNCOMPRESSED_INODES = 0x0001, + SQFS_FLAG_UNCOMPRESSED_DATA = 0x0002, + SQFS_FLAG_UNCOMPRESSED_FRAGMENTS = 0x0008, + SQFS_FLAG_NO_FRAGMENTS = 0x0010, + SQFS_FLAG_ALWAYS_FRAGMENTS = 0x0020, + SQFS_FLAG_DUPLICATES = 0x0040, + SQFS_FLAG_EXPORTABLE = 0x0080, + SQFS_FLAG_UNCOMPRESSED_XATTRS = 0x0100, + SQFS_FLAG_NO_XATTRS = 0x0200, + SQFS_FLAG_COMPRESSOR_OPTIONS = 0x0400, + SQFS_FLAG_UNCOMPRESSED_IDS = 0x0800, +} E_SQFS_SUPER_FLAGS; + +typedef enum { + SQFS_INODE_DIR = 1, + SQFS_INODE_FILE = 2, + SQFS_INODE_SLINK = 3, + SQFS_INODE_BDEV = 4, + SQFS_INODE_CDEV = 5, + SQFS_INODE_FIFO = 6, + SQFS_INODE_SOCKET = 7, + SQFS_INODE_EXT_DIR = 8, + SQFS_INODE_EXT_FILE = 9, + SQFS_INODE_EXT_SLINK = 10, + SQFS_INODE_EXT_BDEV = 11, + SQFS_INODE_EXT_CDEV = 12, + SQFS_INODE_EXT_FIFO = 13, + SQFS_INODE_EXT_SOCKET = 14, +} E_SQFS_INODE_TYPE; + +#endif /* SQUASHFS_H */ diff --git a/sqfs/super.c b/sqfs/super.c new file mode 100644 index 0000000..0241866 --- /dev/null +++ b/sqfs/super.c @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: ISC */ +#include +#include +#include + +#include "util/util.h" +#include "pkg2sqfs.h" + +int sqfs_super_init(sqfs_super_t *s, int64_t timestamp, + uint32_t blocksize, E_SQFS_COMPRESSOR comp) +{ + memset(s, 0, sizeof(*s)); + + if (blocksize & (blocksize - 1)) { + fputs("Block size must be a power of 2\n", stderr); + return -1; + } + + if (blocksize < 4096 || blocksize >= (1 << 24)) { + fputs("Block size must be between 4k and 1M\n", stderr); + return -1; + } + + if (timestamp < 0 || timestamp > 0x00000000FFFFFFFF) { + fputs("Timestamp too large for squashfs\n", stderr); + return -1; + } + + s->magic = SQFS_MAGIC; + s->modification_time = timestamp; + s->block_size = blocksize; + s->compression_id = comp; + s->flags = SQFS_FLAG_NO_FRAGMENTS | SQFS_FLAG_NO_XATTRS; + s->version_major = SQFS_VERSION_MAJOR; + s->version_minor = SQFS_VERSION_MINOR; + s->bytes_used = sizeof(*s); + s->id_table_start = 0xFFFFFFFFFFFFFFFFUL; + s->xattr_id_table_start = 0xFFFFFFFFFFFFFFFFUL; + s->inode_table_start = 0xFFFFFFFFFFFFFFFFUL; + s->directory_table_start = 0xFFFFFFFFFFFFFFFFUL; + s->fragment_table_start = 0xFFFFFFFFFFFFFFFFUL; + s->export_table_start = 0xFFFFFFFFFFFFFFFFUL; + + while (blocksize != 0) { + s->block_log += 1; + blocksize >>= 1; + } + + s->block_log -= 1; + return 0; +} + +int sqfs_super_write(sqfs_info_t *info) +{ + sqfs_super_t copy; + ssize_t ret; + off_t off; + + copy.magic = htole32(info->super.magic); + copy.inode_count = htole32(info->super.inode_count); + copy.modification_time = htole32(info->super.modification_time); + copy.block_size = htole32(info->super.block_size); + copy.fragment_entry_count = htole32(info->super.fragment_entry_count); + copy.compression_id = htole16(info->super.compression_id); + copy.block_log = htole16(info->super.block_log); + copy.flags = htole16(info->super.flags); + copy.id_count = htole16(info->super.id_count); + copy.version_major = htole16(info->super.version_major); + copy.version_minor = htole16(info->super.version_minor); + copy.root_inode_ref = htole64(info->super.root_inode_ref); + copy.bytes_used = htole64(info->super.bytes_used); + copy.id_table_start = htole64(info->super.id_table_start); + copy.xattr_id_table_start = htole64(info->super.xattr_id_table_start); + copy.inode_table_start = htole64(info->super.inode_table_start); + copy.directory_table_start = htole64(info->super.directory_table_start); + copy.fragment_table_start = htole64(info->super.fragment_table_start); + copy.export_table_start = htole64(info->super.export_table_start); + + off = lseek(info->outfd, 0, SEEK_SET); + if (off == (off_t)-1) { + perror("seek on output file"); + return -1; + } + + ret = write_retry(info->outfd, ©, sizeof(copy)); + + if (ret < 0) { + perror("Error writing squashfs super block"); + return -1; + } + + if (ret == 0) { + fputs("Truncated write trying to write squashfs super block\n", + stderr); + return -1; + } + + return 0; +} diff --git a/sqfs/table.c b/sqfs/table.c new file mode 100644 index 0000000..64a0dff --- /dev/null +++ b/sqfs/table.c @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: ISC */ +#include "pkg2sqfs.h" + +int sqfs_write_table(sqfs_info_t *info, const void *data, size_t entsize, + size_t count, uint64_t *startblock) +{ + size_t ent_per_blocks = SQFS_META_BLOCK_SIZE / entsize; + uint64_t blocks[count / ent_per_blocks + 1]; + size_t i, blkidx = 0, tblsize; + meta_writer_t *m; + ssize_t ret; + + /* Write actual data. Whenever we cross a block boundary, remember + the block start offset */ + m = meta_writer_create(info->outfd); + if (m == NULL) + return -1; + + for (i = 0; i < count; ++i) { + if (blkidx == 0 || m->written > blocks[blkidx - 1]) + blocks[blkidx++] = m->written; + + if (meta_writer_append(m, data, entsize)) + goto fail; + + data = (const char *)data + entsize; + } + + if (meta_writer_flush(m)) + goto fail; + + for (i = 0; i < blkidx; ++i) + blocks[i] = htole64(blocks[i] + info->super.bytes_used); + + info->super.bytes_used += m->written; + meta_writer_destroy(m); + + /* write new index table */ + *startblock = info->super.bytes_used; + tblsize = sizeof(blocks[0]) * blkidx; + + ret = write_retry(info->outfd, blocks, tblsize); + if (ret < 0) { + perror("writing index table"); + return -1; + } + + if ((size_t)ret < tblsize) { + fputs("index table truncated\n", stderr); + return -1; + } + + info->super.bytes_used += tblsize; + return 0; +fail: + meta_writer_destroy(m); + return -1; +} diff --git a/sqfs/vfs.c b/sqfs/vfs.c new file mode 100644 index 0000000..ad6cb26 --- /dev/null +++ b/sqfs/vfs.c @@ -0,0 +1,322 @@ +/* SPDX-License-Identifier: ISC */ +#include +#include +#include +#include + +#include "pkg2sqfs.h" + +static int id_to_index(vfs_t *fs, uint32_t id, uint16_t *out) +{ + size_t i, new_sz; + void *new; + + for (i = 0; i < fs->num_ids; ++i) { + if (fs->id_tbl[i] == id) { + *out = i; + return 0; + } + } + + if (fs->num_ids > 0xFFFF) { + fputs("too many unique UIDs/GIDs (more than 64k)!\n", stderr); + return -1; + } + + if (fs->num_ids == fs->max_ids) { + new_sz = fs->max_ids + 10; + new = realloc(fs->id_tbl, new_sz * sizeof(fs->id_tbl[0])); + + if (new == NULL) { + perror("expanding ID table"); + return -1; + } + + fs->max_ids = new_sz; + fs->id_tbl = new; + } + + *out = fs->num_ids; + fs->id_tbl[fs->num_ids++] = id; + return 0; +} + +static node_t *node_create(vfs_t *fs, node_t *parent, size_t extra, + const char *name, size_t name_len, + uint16_t mode, uint32_t uid, uint32_t gid) +{ + node_t *n = calloc(1, sizeof(*n) + extra + name_len + 1); + + if (n == NULL) { + perror("allocating vfs node"); + return NULL; + } + + if (parent != NULL) { + n->parent = parent; + n->next = parent->data.dir->children; + parent->data.dir->children = n; + } + + if (id_to_index(fs, uid, &n->uid_idx)) + return NULL; + + if (id_to_index(fs, gid, &n->gid_idx)) + return NULL; + + n->mode = mode; + n->name = (char *)n->extra; + memcpy(n->extra, name, name_len); + return n; +} + +static node_t *node_from_ent(vfs_t *fs, sqfs_super_t *s, image_entry_t *ent) +{ + node_t *n, *parent = fs->root; + const char *path = ent->name; + size_t len, extra; + char *end; +next: + end = strchrnul(path, '/'); + len = end - path; + + for (n = parent->data.dir->children; n != NULL; n = n->next) { + if (!strncmp(n->name, path, len) && strlen(n->name) == len) { + if (path[len] != '/') + goto fail_exists; + if (!S_ISDIR(n->mode)) + goto fail_not_dir; + parent = n; + path += len + 1; + goto next; + } + } + + if (path[len] == '/') { + if (S_ISDIR(ent->mode)) { + n = node_create(fs, parent, sizeof(dir_info_t), + path, len, S_IFDIR | fs->default_mode, + fs->default_uid, fs->default_gid); + if (n == NULL) + return NULL; + + n->data.dir = + (dir_info_t *)(n->name + strlen(n->name) + 1); + + parent = n; + path += len + 1; + goto next; + } + goto fail_no_ent; + } + + switch (ent->mode & S_IFMT) { + case S_IFREG: + extra = sizeof(file_info_t); + + extra += (ent->data.file.size / s->block_size) * + sizeof(uint32_t); + break; + case S_IFLNK: + extra = strlen(ent->data.symlink.target) + 1; + break; + case S_IFDIR: + extra = sizeof(dir_info_t); + break; + default: + extra = 0; + break; + } + + n = node_create(fs, parent, extra, path, len, + ent->mode, ent->uid, ent->gid); + if (n == NULL) + return NULL; + + switch (ent->mode & S_IFMT) { + case S_IFREG: + n->data.file = (file_info_t *)(n->name + strlen(n->name) + 1); + n->data.file->size = ent->data.file.size; + n->data.file->id = ent->data.file.id; + n->data.file->startblock = 0xFFFFFFFFFFFFFFFFUL; + n->data.file->fragment = 0xFFFFFFFFL; + break; + case S_IFLNK: + n->data.symlink = n->name + strlen(n->name) + 1; + strcpy((char *)n->data.symlink, ent->data.symlink.target); + break; + case S_IFBLK: + case S_IFCHR: + n->data.device = ent->data.device.devno; + break; + case S_IFDIR: + n->data.dir = (dir_info_t *)(n->name + strlen(n->name) + 1); + break; + default: + break; + } + + return n; +fail_no_ent: + fprintf(stderr, "cannot create /%s: '%.*s' does not exist\n", + ent->name, (int)len, path); + return NULL; +fail_exists: + fprintf(stderr, "cannot create /%s: already exists\n", ent->name); + return NULL; +fail_not_dir: + fprintf(stderr, "cannot create /%s: %.*s is not a directory\n", + ent->name, (int)len, path); + return NULL; +} + +static void node_recursive_delete(node_t *n) +{ + node_t *child; + + if (n != NULL && S_ISDIR(n->mode)) { + while (n->data.dir->children != NULL) { + child = n->data.dir->children; + n->data.dir->children = child->next; + + node_recursive_delete(child); + } + } + + free(n); +} + +typedef int(*node_cb)(vfs_t *fs, node_t *n, void *user); + +static int foreach_node(vfs_t *fs, node_t *root, void *user, node_cb cb) +{ + bool has_subdirs = false; + node_t *it; + + for (it = root->data.dir->children; it != NULL; it = it->next) { + if (S_ISDIR(it->mode)) { + has_subdirs = true; + break; + } + } + + if (has_subdirs) { + for (it = root->data.dir->children; it != NULL; it = it->next) { + if (!S_ISDIR(it->mode)) + continue; + + if (foreach_node(fs, it, user, cb) != 0) + return -1; + } + } + + for (it = root->data.dir->children; it != NULL; it = it->next) { + if (cb(fs, it, user)) + return -1; + } + + if (root->parent == NULL) { + if (cb(fs, root, user)) + return -1; + } + + return 0; +} + +static int alloc_inum(vfs_t *fs, node_t *n, void *user) +{ + uint32_t *counter = user; + (void)fs; + + if (*counter == 0xFFFFFFFF) { + fputs("too many inodes (more than 2^32 - 2)!\n", stderr); + return -1; + } + + n->inode_num = (*counter)++; + return 0; +} + +static int link_node_to_tbl(vfs_t *fs, node_t *n, void *user) +{ + (void)user; + fs->node_tbl[n->inode_num] = n; + return 0; +} + +int create_vfs_tree(sqfs_info_t *info) +{ + image_entry_t *ent, *list; + uint32_t counter = 2; + vfs_t *fs = &info->fs; + node_t *n; + size_t i; + + if (image_entry_list_from_package(info->rd, &list)) + return -1; + + fs->root = node_create(fs, NULL, sizeof(dir_info_t), + "", 0, S_IFDIR | 0755, 0, 0); + if (fs->root == NULL) + goto fail; + + fs->root->data.dir = (dir_info_t *)(fs->root->name + strlen(fs->root->name) + 1); + + for (ent = list; ent != NULL; ent = ent->next) { + n = node_from_ent(fs, &info->super, ent); + if (n == NULL) + goto fail; + } + + if (foreach_node(fs, fs->root, &counter, alloc_inum)) + goto fail; + + fs->num_inodes = counter; + fs->node_tbl = calloc(fs->num_inodes, sizeof(fs->node_tbl[0])); + if (fs->node_tbl == NULL) + goto fail_oom; + + if (foreach_node(fs, fs->root, NULL, link_node_to_tbl)) + goto fail; + + info->super.inode_count = fs->num_inodes - 2; + info->super.id_count = fs->num_ids; + + image_entry_free_list(list); + + for (i = 0; i < fs->num_ids; ++i) + fs->id_tbl[i] = htole32(fs->id_tbl[i]); + + return 0; +fail_oom: + fputs("out of memory\n", stderr); +fail: + image_entry_free_list(list); + free(fs->node_tbl); + free(fs->id_tbl); + node_recursive_delete(fs->root); + return -1; +} + +void destroy_vfs_tree(vfs_t *fs) +{ + free(fs->node_tbl); + free(fs->id_tbl); + node_recursive_delete(fs->root); +} + +node_t *vfs_node_from_file_id(vfs_t *fs, uint32_t id) +{ + size_t i; + + for (i = 0; i < fs->num_inodes; ++i) { + if (fs->node_tbl[i] == NULL) + continue; + if (!S_ISREG(fs->node_tbl[i]->mode)) + continue; + if (fs->node_tbl[i]->data.file->id == id) + return fs->node_tbl[i]; + } + + return NULL; +}