pygos/pkg-utils
1
0
Fork 0
mirror of https://github.com/pygos/pkg-utils.git synced 2024-11-21 12:29:47 +01:00
Code Issues Releases Wiki Activity

Add pkg2sqfs utility stub

Browse source 
The goal is to transform a package file to a squashfs image. The mksquashfs
utility is a PITA with all kinds of idiotic not-thought-through corner
cases (e.g. the way "pseudo file definitions" have been tacked on, the
inabillity to set UID/GID for / *AND* have files not owned by root, the
way xattrs are handled, .....).

It would be preferable to fix mksquashfs itself, but the source code itself
is a horrid, unmaintained garbage pile. Its easier to write a small utility
instead that can turn a pkg file into a squashfs image in a way that
propperly treats file permissions and ownership, and is deterministic
(i.e. reproducable).

The utility currently generates an uncompressed squashfs image that
unsquashfs can unpack, but the kernel refuses the mount. The exact
problem still needs to be determined.

Furthermore, compression has to be implemented and some cleanup shoul
be done.

Signed-off-by: David Oberhollenzer <goliath@infraroot.at>
This commit is contained in:
David Oberhollenzer 2019-04-05 14:19:23 +02:00
parent c78dc2255f
commit 858779e42c
13 changed files with 1584 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -18,3 +18,4 @@ config.h
*.a
*~
pkg
pkg2sqfs

1
Makefile.am
View file

@ -10,3 +10,4 @@ EXTRA_DIST = autogen.sh LICENSE README.md
include lib/Makemodule.am
include main/Makemodule.am
include sqfs/Makemodule.am

4
configure.ac
View file

@ -50,6 +50,10 @@ PKG_CHECK_MODULES(XZ, [liblzma >= 5.0.0], [have_lzma="yes"], [])
AM_CONDITIONAL([WITH_ZLIB], [test "x$have_zlib" == "xyes"])
AM_CONDITIONAL([WITH_LZMA], [test "x$have_lzma" == "xyes"])
AM_CONDITIONAL([HAVE_LIBBSD], [true])
PKG_CHECK_MODULES(LIBBSD, [libbsd], [],
[AC_MSG_ERROR([missing libbsd])])
##### generate output #####
AC_CONFIG_HEADERS([config.h])

22
sqfs/Makemodule.am Normal file
View file

@ -0,0 +1,22 @@
pkg2sqfs_SOURCES = sqfs/squashfs.h sqfs/super.c sqfs/pkg2sqfs.c
pkg2sqfs_SOURCES += sqfs/vfs.c sqfs/pkg2sqfs.h sqfs/block.c
pkg2sqfs_SOURCES += sqfs/meta.c sqfs/meta_writer.c sqfs/table.c
pkg2sqfs_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/sqfs
pkg2sqfs_CFLAGS = $(AM_CFLAGS)
pkg2sqfs_LDADD = libpkg.a libutil.a libfilelist.a libcomp.a
if WITH_LZMA
pkg2sqfs_LDADD += $(XZ_LIBS)
endif
if WITH_ZLIB
pkg2sqfs_LDADD += $(ZLIB_LIBS)
endif
if HAVE_LIBBSD
pkg2sqfs_CPPFLAGS += -DHAVE_LIBBSD
pkg2sqfs_CFLAGS += $(LIBBSD_CFLAGS)
pkg2sqfs_LDADD += $(LIBBSD_LIBS)
endif
bin_PROGRAMS += pkg2sqfs

212
sqfs/block.c Normal file
View file

@ -0,0 +1,212 @@
/* SPDX-License-Identifier: ISC */
#include "pkg2sqfs.h"
static int write_block(node_t *node, sqfs_info_t *info)
{
size_t idx, bs;
ssize_t ret;
/* TODO: try to compress the block */
idx = info->file_block_count++;
bs = info->super.block_size;
ret = write_retry(info->outfd, info->block, bs);
if (ret < 0) {
perror("writing to output file");
return -1;
}
if ((size_t)ret < bs) {
fputs("write to output file truncated\n", stderr);
return -1;
}
node->data.file->blocksizes[idx] = bs | (1 << 24);
info->super.flags |= SQFS_FLAG_UNCOMPRESSED_DATA;
info->super.bytes_used += bs;
return 0;
}
static int flush_fragments(sqfs_info_t *info)
{
size_t newsz, size;
file_info_t *fi;
uint64_t offset;
ssize_t ret;
void *new;
if (info->num_fragments == info->max_fragments) {
newsz = info->max_fragments ? info->max_fragments * 2 : 16;
new = realloc(info->fragments,
sizeof(info->fragments[0]) * newsz);
if (new == NULL) {
perror("appending to fragment table");
return -1;
}
info->max_fragments = newsz;
info->fragments = new;
}
offset = info->super.bytes_used;
size = info->frag_offset;
for (fi = info->frag_list; fi != NULL; fi = fi->frag_next)
fi->fragment = info->num_fragments;
info->fragments[info->num_fragments].start_offset = htole64(offset);
info->fragments[info->num_fragments].size = htole32(size | (1 << 24));
info->fragments[info->num_fragments].pad0 = 0;
info->num_fragments += 1;
/* TODO: try to compress the fragments */
ret = write_retry(info->outfd, info->fragment, size);
if (ret < 0) {
perror("writing to output file");
return -1;
}
if ((size_t)ret < size) {
fputs("write to output file truncated\n", stderr);
return -1;
}
memset(info->fragment, 0, info->super.block_size);
info->super.bytes_used += size;
info->frag_offset = 0;
info->frag_list = NULL;
info->super.flags &= ~SQFS_FLAG_NO_FRAGMENTS;
info->super.flags |= SQFS_FLAG_UNCOMPRESSED_FRAGMENTS;
info->super.flags |= SQFS_FLAG_ALWAYS_FRAGMENTS;
return 0;
}
static int add_fragment(file_info_t *fi, sqfs_info_t *info, size_t size)
{
if (info->frag_offset + size > info->super.block_size) {
if (flush_fragments(info))
return -1;
}
fi->fragment_offset = info->frag_offset;
fi->frag_next = info->frag_list;
info->frag_list = fi;
memcpy((char *)info->fragment + info->frag_offset, info->block, size);
info->frag_offset += size;
return 0;
}
static int process_file(node_t *node, sqfs_info_t *info)
{
uint64_t count = node->data.file->size;
int ret;
size_t diff;
node->data.file->startblock = info->super.bytes_used;
info->file_block_count = 0;
while (count != 0) {
diff = count > (uint64_t)info->super.block_size ?
info->super.block_size : count;
ret = pkg_reader_read_payload(info->rd, info->block, diff);
if (ret < 0)
return -1;
if ((size_t)ret < diff)
goto fail_trunc;
if (diff < info->super.block_size) {
if (add_fragment(node->data.file, info, diff))
return -1;
} else {
if (write_block(node, info))
return -1;
}
count -= diff;
}
return 0;
fail_trunc:
fprintf(stderr, "%s: truncated data block\n",
pkg_reader_get_filename(info->rd));
return -1;
}
int pkg_data_to_sqfs(sqfs_info_t *info)
{
file_data_t frec;
record_t *hdr;
node_t *node;
int ret;
if (pkg_reader_rewind(info->rd))
return -1;
memset(info->fragment, 0, info->super.block_size);
for (;;) {
ret = pkg_reader_get_next_record(info->rd);
if (ret == 0)
break;
if (ret < 0)
return -1;
hdr = pkg_reader_current_record_header(info->rd);
if (hdr->magic != PKG_MAGIC_DATA)
continue;
for (;;) {
ret = pkg_reader_read_payload(info->rd, &frec,
sizeof(frec));
if (ret == 0)
break;
if (ret < 0)
return -1;
if ((size_t)ret < sizeof(frec))
goto fail_trunc;
frec.id = le32toh(frec.id);
node = vfs_node_from_file_id(&info->fs, frec.id);
if (node == NULL)
goto fail_meta;
if (process_file(node, info))
return -1;
}
}
if (info->frag_offset != 0) {
if (flush_fragments(info))
return -1;
}
return 0;
fail_meta:
fprintf(stderr, "%s: missing meta information for file %u\n",
pkg_reader_get_filename(info->rd), (unsigned int)frec.id);
return -1;
fail_trunc:
fprintf(stderr, "%s: truncated file data record\n",
pkg_reader_get_filename(info->rd));
return -1;
}
int sqfs_write_fragment_table(sqfs_info_t *info)
{
info->super.fragment_entry_count = info->num_fragments;
return sqfs_write_table(info, info->fragments,
sizeof(info->fragments[0]),
info->num_fragments,
&info->super.fragment_table_start);
}

305
sqfs/meta.c Normal file
View file

@ -0,0 +1,305 @@
/* SPDX-License-Identifier: ISC */
#include "pkg2sqfs.h"
static size_t hard_link_count(node_t *node)
{
size_t count;
node_t *n;
if (S_ISDIR(node->mode)) {
count = 2;
for (n = node->data.dir->children; n != NULL; n = n->next)
++count;
return count;
}
return 1;
}
static int write_dir(meta_writer_t *dm, node_t *node)
{
sqfs_dir_header_t hdr;
sqfs_dir_entry_t ent;
size_t i, count;
node_t *c, *d;
c = node->data.dir->children;
node->data.dir->size = 0;
node->data.dir->dref = (dm->written << 16) | dm->offset;
while (c != NULL) {
count = 0;
for (d = c; d != NULL; d = d->next) {
if ((d->inode_ref >> 16) != (c->inode_ref >> 16))
break;
if ((d->inode_num - c->inode_num) > 0xFFFF)
break;
count += 1;
}
hdr.count = htole32(count - 1);
hdr.start_block = htole32(c->inode_ref >> 16);
hdr.inode_number = htole32(c->inode_num);
if (meta_writer_append(dm, &hdr, sizeof(hdr)))
return -1;
d = c;
for (i = 0; i < count; ++i) {
ent.offset = htole16(c->inode_ref & 0x0000FFFF);
ent.inode_number = htole16(c->inode_num - d->inode_num);
ent.type = htole16(c->type);
ent.size = htole16(strlen(c->name) - 1);
if (meta_writer_append(dm, &ent, sizeof(ent)))
return -1;
if (meta_writer_append(dm, c->name, strlen(c->name)))
return -1;
c = c->next;
}
node->data.dir->size += sizeof(hdr) + count * sizeof(ent);
}
return 0;
}
static int write_inode(sqfs_info_t *info, meta_writer_t *im, meta_writer_t *dm,
node_t *node)
{
file_info_t *fi = NULL;
sqfs_inode_t base;
uint32_t bs;
uint64_t i;
node->inode_ref = (im->written << 16) | im->offset;
base.mode = htole16(node->mode);
base.uid_idx = htole16(node->uid_idx);
base.gid_idx = htole16(node->gid_idx);
base.mod_time = htole32(info->super.modification_time);
base.inode_number = htole32(node->inode_num);
switch (node->mode & S_IFMT) {
case S_IFLNK: node->type = SQFS_INODE_SLINK; break;
case S_IFBLK: node->type = SQFS_INODE_BDEV; break;
case S_IFCHR: node->type = SQFS_INODE_CDEV; break;
case S_IFDIR:
if (write_dir(dm, node))
return -1;
node->type = SQFS_INODE_DIR;
i = node->data.dir->dref;
if ((i >> 16) > 0xFFFFFFFFUL || node->data.dir->size > 0xFFFF)
node->type = SQFS_INODE_EXT_DIR;
break;
case S_IFREG:
fi = node->data.file;
node->type = SQFS_INODE_FILE;
if (fi->startblock > 0xFFFFFFFFUL || fi->size > 0xFFFFFFFFUL ||
hard_link_count(node) > 1) {
node->type = SQFS_INODE_EXT_FILE;
}
break;
default:
assert(0);
}
base.type = htole16(node->type);
if (meta_writer_append(im, &base, sizeof(base)))
return -1;
switch (node->type) {
case SQFS_INODE_SLINK: {
sqfs_inode_slink_t slink = {
.nlink = htole32(hard_link_count(node)),
.target_size = htole32(strlen(node->data.symlink)),
};
if (meta_writer_append(im, &slink, sizeof(slink)))
return -1;
if (meta_writer_append(im, node->data.symlink,
le32toh(slink.target_size))) {
return -1;
}
break;
}
case SQFS_INODE_BDEV:
case SQFS_INODE_CDEV: {
sqfs_inode_dev_t dev = {
.nlink = htole32(hard_link_count(node)),
.devno = htole32(node->data.device),
};
return meta_writer_append(im, &dev, sizeof(dev));
}
case SQFS_INODE_EXT_FILE: {
sqfs_inode_file_ext_t ext = {
.blocks_start = htole64(fi->startblock),
.file_size = htole64(fi->size),
.sparse = htole64(0xFFFFFFFFFFFFFFFFUL),
.nlink = htole32(hard_link_count(node)),
.fragment_idx = htole32(fi->fragment),
.fragment_offset = htole32(fi->fragment_offset),
.xattr_idx = htole32(0xFFFFFFFF),
};
if (meta_writer_append(im, &ext, sizeof(ext)))
return -1;
goto out_file_blocks;
}
case SQFS_INODE_FILE: {
sqfs_inode_file_t reg = {
.blocks_start = htole32(fi->startblock),
.fragment_index = htole32(fi->fragment),
.fragment_offset = htole32(fi->fragment_offset),
.file_size = htole32(fi->size),
};
if (meta_writer_append(im, &reg, sizeof(reg)))
return -1;
goto out_file_blocks;
}
case SQFS_INODE_DIR: {
sqfs_inode_dir_t dir = {
.start_block = htole32(node->data.dir->dref >> 16),
.nlink = htole32(hard_link_count(node)),
.size = htole16(node->data.dir->size),
.offset = htole16(node->data.dir->dref & 0xFFFF),
.parent_inode = node->parent ?
htole32(node->parent->inode_num) : htole32(1),
};
return meta_writer_append(im, &dir, sizeof(dir));
}
case SQFS_INODE_EXT_DIR: {
sqfs_inode_dir_ext_t ext = {
.nlink = htole32(hard_link_count(node)),
.size = htole32(node->data.dir->size),
.start_block = htole32(node->data.dir->dref >> 16),
.parent_inode = node->parent ?
htole32(node->parent->inode_num) : htole32(1),
.inodex_count = htole32(0),
.offset = htole16(node->data.dir->dref & 0xFFFF),
.xattr_idx = htole32(0xFFFFFFFF),
};
return meta_writer_append(im, &ext, sizeof(ext));
}
default:
assert(0);
}
return 0;
out_file_blocks:
for (i = 0; i < fi->size / info->super.block_size; ++i) {
bs = htole32(fi->blocksizes[i]);
if (meta_writer_append(im, &bs, sizeof(bs)))
return -1;
}
return 0;
}
int sqfs_write_inodes(sqfs_info_t *info)
{
meta_writer_t *im, *dm;
size_t i, diff;
ssize_t ret;
FILE *tmp;
int tmpfd;
tmp = tmpfile();
if (tmp == NULL) {
perror("tmpfile");
return -1;
}
tmpfd = fileno(tmp);
im = meta_writer_create(info->outfd);
if (im == NULL)
goto fail_tmp;
dm = meta_writer_create(tmpfd);
if (dm == NULL)
goto fail_im;
for (i = 2; i < info->fs.num_inodes; ++i) {
if (write_inode(info, im, dm, info->fs.node_tbl[i]))
goto fail;
}
if (meta_writer_flush(im))
goto fail;
if (meta_writer_flush(dm))
goto fail;
info->super.root_inode_ref = info->fs.root->inode_ref;
info->super.inode_table_start = info->super.bytes_used;
info->super.bytes_used += im->written;
info->super.directory_table_start = info->super.bytes_used;
info->super.bytes_used += dm->written;
if (lseek(tmpfd, 0, SEEK_SET) == (off_t)-1) {
perror("rewind on directory temp file");
goto fail;
}
for (;;) {
ret = read_retry(tmpfd, dm->data, sizeof(dm->data));
if (ret < 0) {
perror("read from temp file");
goto fail;
}
if (ret == 0)
break;
diff = ret;
ret = write_retry(info->outfd, dm->data, diff);
if (ret < 0) {
perror("write to image file");
goto fail;
}
if ((size_t)ret < diff) {
fputs("copying meta data to image file: "
"truncated write\n", stderr);
goto fail;
}
}
info->super.flags |= SQFS_FLAG_UNCOMPRESSED_INODES;
meta_writer_destroy(dm);
meta_writer_destroy(im);
fclose(tmp);
return 0;
fail:
meta_writer_destroy(dm);
fail_im:
meta_writer_destroy(im);
fail_tmp:
fclose(tmp);
return -1;
}
int sqfs_write_ids(sqfs_info_t *info)
{
info->super.flags |= SQFS_FLAG_UNCOMPRESSED_IDS;
return sqfs_write_table(info, info->fs.id_tbl,
sizeof(info->fs.id_tbl[0]),
info->fs.num_ids, &info->super.id_table_start);
}

74
sqfs/meta_writer.c Normal file
View file

@ -0,0 +1,74 @@
/* SPDX-License-Identifier: ISC */
#include "pkg2sqfs.h"
meta_writer_t *meta_writer_create(int fd)
{
meta_writer_t *m = calloc(1, sizeof(*m));
if (m == NULL) {
perror("creating meta data writer");
return NULL;
}
m->outfd = fd;
return m;
}
void meta_writer_destroy(meta_writer_t *m)
{
free(m);
}
int meta_writer_flush(meta_writer_t *m)
{
ssize_t ret, count;
/* TODO: compress buffer */
if (m->offset == 0)
return 0;
((uint16_t *)m->data)[0] = htole16(m->offset | 0x8000);
count = m->offset + 2;
ret = write_retry(m->outfd, m->data, count);
if (ret < 0) {
perror("writing meta data");
return -1;
}
if (ret < count) {
fputs("meta data was truncated\n", stderr);
return -1;
}
memset(m->data, 0, sizeof(m->data));
m->offset = 0;
m->written += count;
return 0;
}
int meta_writer_append(meta_writer_t *m, const void *data, size_t size)
{
size_t diff;
while (size != 0) {
diff = sizeof(m->data) - 2 - m->offset;
if (diff == 0) {
if (meta_writer_flush(m))
return -1;
diff = sizeof(m->data) - 2;
}
if (diff > size)
diff = size;
memcpy(m->data + 2 + m->offset, data, diff);
m->offset += diff;
size -= diff;
data = (const char *)data + diff;
}
return 0;
}

191
sqfs/pkg2sqfs.c Normal file
View file

@ -0,0 +1,191 @@
/* SPDX-License-Identifier: ISC */
#include "pkg2sqfs.h"
static struct option long_opts[] = {
{ "block-size", required_argument, NULL, 'b' },
{ "force", no_argument, NULL, 'f' },
{ "version", no_argument, NULL, 'V' },
{ "help", no_argument, NULL, 'h' },
};
static const char *short_opts = "b:fhV";
extern char *__progname;
static const char *version_string =
"%s (Pygos %s) %s\n"
"Copyright (c) 2019 David Oberhollenzer\n"
"License ISC <https://opensource.org/licenses/ISC>\n"
"This is free software: you are free to change and redistribute it.\n"
"There is NO WARRANTY, to the extent permitted by law.\n"
"\n"
"Written by David Oberhollenzer.\n";
static const char *help_string =
"Usage: %s [OPTIONS] <package-file> <squashfs-file>\n"
"\n"
"Convert a package file into a squashfs image.\n"
"\n"
"Possible options:\n"
"\n"
" --block-size, -b <size> Block size to use for Squashfs image.\n"
" Defaults to %u.\n"
" --force, -f Overwrite the output file if it already exists.\n"
" --help, -h Print help text and exit.\n"
" --version, -V Print version information and exit.\n"
"\n";
static void print_tree(int level, node_t *n)
{
int i;
for (i = 0; i < (level - 1); ++i)
fputs("| ", stdout);
switch (n->mode & S_IFMT) {
case S_IFDIR:
printf("%s%s/ (%u, %o)\n", level ? "+- " : "", n->name,
n->inode_num, (unsigned int)n->mode);
n = n->data.dir->children;
++level;
while (n != NULL) {
print_tree(level, n);
n = n->next;
}
break;
case S_IFLNK:
printf("+- %s (%u) -> %s\n", n->name, n->inode_num,
n->data.symlink);
break;
case S_IFBLK:
printf("+- %s (%u) b %lu\n", n->name, n->inode_num,
n->data.device);
break;
case S_IFCHR:
printf("+- %s (%u) c %lu\n", n->name, n->inode_num,
n->data.device);
break;
default:
printf("+- %s (%u)\n", n->name, n->inode_num);
break;
}
}
int main(int argc, char **argv)
{
uint32_t blocksize = SQFS_DEFAULT_BLOCK_SIZE, timestamp = 0;
int i, outmode = O_WRONLY | O_CREAT | O_EXCL;
const char *infile, *outfile, *errstr;
int status = EXIT_FAILURE;
sqfs_info_t info;
memset(&info, 0, sizeof(info));
for (;;) {
i = getopt_long(argc, argv, short_opts, long_opts, NULL);
if (i == -1)
break;
switch (i) {
case 'b':
blocksize = strtonum(optarg, 1024, 0xFFFFFFFF, &errstr);
if (errstr != NULL) {
fprintf(stderr, "Block size '%s': %s\n",
optarg, errstr);
return EXIT_FAILURE;
}
break;
case 'f':
outmode = O_WRONLY | O_CREAT | O_TRUNC;
break;
case 'h':
printf(help_string, __progname,
SQFS_DEFAULT_BLOCK_SIZE);
exit(EXIT_SUCCESS);
case 'V':
printf(version_string, __progname,
PACKAGE_NAME, PACKAGE_VERSION);
exit(EXIT_SUCCESS);
default:
goto fail_arg;
}
}
if ((optind + 1) >= argc) {
fputs("Missing arguments: input and output files.\n", stderr);
goto fail_arg;
}
infile = argv[optind++];
outfile = argv[optind++];
info.rd = pkg_reader_open(infile);
if (info.rd == NULL)
return EXIT_FAILURE;
info.outfd = open(outfile, outmode, 0644);
if (info.outfd < 0) {
perror(outfile);
goto out_pkg_close;
}
if (sqfs_super_init(&info.super, timestamp, blocksize, SQFS_COMP_GZIP))
goto out_close;
info.block = malloc(info.super.block_size * 3);
if (info.block == NULL) {
perror("malloc");
goto out_close;
}
info.scratch = (char *)info.block + info.super.block_size;
info.fragment = (char *)info.block + 2 * info.super.block_size;
info.fs.default_uid = 0;
info.fs.default_gid = 0;
info.fs.default_mode = 0755;
if (create_vfs_tree(&info))
goto out_buffer;
print_tree(0, info.fs.root);
if (sqfs_super_write(&info))
goto out_tree;
if (pkg_data_to_sqfs(&info))
goto out_fragments;
free(info.block);
info.block = NULL;
if (sqfs_write_inodes(&info))
goto out_fragments;
if (sqfs_write_fragment_table(&info))
goto out_fragments;
if (sqfs_write_ids(&info))
goto out_fragments;
if (sqfs_super_write(&info))
goto out_fragments;
status = EXIT_SUCCESS;
out_fragments:
free(info.fragments);
out_tree:
destroy_vfs_tree(&info.fs);
out_buffer:
free(info.block);
out_close:
close(info.outfd);
out_pkg_close:
pkg_reader_close(info.rd);
return status;
fail_arg:
fprintf(stderr, "Try `%s --help' for more information.\n", __progname);
return EXIT_FAILURE;
}

137
sqfs/pkg2sqfs.h Normal file
View file

@ -0,0 +1,137 @@
/* SPDX-License-Identifier: ISC */
#ifndef PKG2SQFS_H
#define PKG2SQFS_H
#include <sys/types.h>
#include <stdbool.h>
#include <unistd.h>
#include <stdlib.h>
#include <getopt.h>
#include <stdint.h>
#include <assert.h>
#include <stdio.h>
#include <fcntl.h>
#include <errno.h>
#ifdef HAVE_LIBBSD
#include <bsd/bsd.h>
#endif
#include "util/util.h"
#include "filelist/image_entry.h"
#include "pkg/pkgreader.h"
#include "pkg/pkgio.h"
#include "squashfs.h"
#include "config.h"
typedef struct {
uint8_t data[SQFS_META_BLOCK_SIZE + 2];
size_t offset;
size_t written;
int outfd;
} meta_writer_t;
typedef struct file_info_t {
struct file_info_t *frag_next;
uint64_t size;
uint64_t startblock;
uint32_t id;
uint32_t fragment;
uint32_t fragment_offset;
uint32_t blocksizes[];
} file_info_t;
typedef struct {
struct node_t *children;
uint64_t dref;
size_t size;
} dir_info_t;
typedef struct node_t {
struct node_t *parent;
struct node_t *next;
const char *name;
union {
dir_info_t *dir;
const char *symlink;
dev_t device;
file_info_t *file;
} data;
uint64_t inode_ref;
uint32_t inode_num;
uint16_t type;
uint16_t mode;
uint16_t uid_idx;
uint16_t gid_idx;
uint8_t extra[];
} node_t;
typedef struct {
uint32_t default_uid;
uint32_t default_gid;
uint32_t default_mode;
node_t **node_tbl;
size_t num_inodes;
uint32_t *id_tbl;
size_t num_ids;
size_t max_ids;
node_t *root;
} vfs_t;
typedef struct {
int outfd;
sqfs_super_t super;
vfs_t fs;
pkg_reader_t *rd;
void *block;
void *scratch;
void *fragment;
sqfs_fragment_t *fragments;
size_t num_fragments;
size_t max_fragments;
int file_block_count;
file_info_t *frag_list;
size_t frag_offset;
} sqfs_info_t;
int pkg_data_to_sqfs(sqfs_info_t *info);
int sqfs_super_init(sqfs_super_t *s, int64_t timestamp,
uint32_t blocksize, E_SQFS_COMPRESSOR comp);
int sqfs_super_write(sqfs_info_t *info);
int create_vfs_tree(sqfs_info_t *info);
void destroy_vfs_tree(vfs_t *fs);
node_t *vfs_node_from_file_id(vfs_t *fs, uint32_t id);
int sqfs_write_inodes(sqfs_info_t *info);
int sqfs_write_ids(sqfs_info_t *info);
int sqfs_write_fragment_table(sqfs_info_t *info);
meta_writer_t *meta_writer_create(int fd);
void meta_writer_destroy(meta_writer_t *m);
int meta_writer_flush(meta_writer_t *m);
int meta_writer_append(meta_writer_t *m, const void *data, size_t size);
int sqfs_write_table(sqfs_info_t *info, const void *data, size_t entsize,
size_t count, uint64_t *start);
#endif /* PKG2SQFS_H */

158
sqfs/squashfs.h Normal file
View file

@ -0,0 +1,158 @@
/* SPDX-License-Identifier: ISC */
#ifndef SQUASHFS_H
#define SQUASHFS_H
#include <stdint.h>
#define SQFS_MAGIC 0x73717368
#define SQFS_VERSION_MAJOR 4
#define SQFS_VERSION_MINOR 0
#define SQFS_META_BLOCK_SIZE 8192
#define SQFS_DEFAULT_BLOCK_SIZE 131072
typedef struct {
uint32_t magic;
uint32_t inode_count;
uint32_t modification_time;
uint32_t block_size;
uint32_t fragment_entry_count;
uint16_t compression_id;
uint16_t block_log;
uint16_t flags;
uint16_t id_count;
uint16_t version_major;
uint16_t version_minor;
uint64_t root_inode_ref;
uint64_t bytes_used;
uint64_t id_table_start;
uint64_t xattr_id_table_start;
uint64_t inode_table_start;
uint64_t directory_table_start;
uint64_t fragment_table_start;
uint64_t export_table_start;
} __attribute__((packed)) sqfs_super_t;
typedef struct {
uint64_t start_offset;
uint32_t size;
uint32_t pad0;
} sqfs_fragment_t;
typedef struct {
uint16_t type;
uint16_t mode;
uint16_t uid_idx;
uint16_t gid_idx;
uint32_t mod_time;
uint32_t inode_number;
} sqfs_inode_t;
typedef struct {
uint32_t nlink;
uint32_t devno;
} sqfs_inode_dev_t;
typedef struct {
uint32_t nlink;
uint32_t devno;
uint32_t xattr_idx;
} sqfs_inode_dev_ext_t;
typedef struct {
uint32_t nlink;
uint32_t target_size;
uint8_t target[];
} sqfs_inode_slink_t;
typedef struct {
uint32_t blocks_start;
uint32_t fragment_index;
uint32_t fragment_offset;
uint32_t file_size;
uint32_t block_sizes[];
} sqfs_inode_file_t;
typedef struct {
uint64_t blocks_start;
uint64_t file_size;
uint64_t sparse;
uint32_t nlink;
uint32_t fragment_idx;
uint32_t fragment_offset;
uint32_t xattr_idx;
uint32_t block_sizes[];
} sqfs_inode_file_ext_t;
typedef struct {
uint32_t start_block;
uint32_t nlink;
uint16_t size;
uint16_t offset;
uint32_t parent_inode;
} sqfs_inode_dir_t;
typedef struct {
uint32_t nlink;
uint32_t size;
uint32_t start_block;
uint32_t parent_inode;
uint16_t inodex_count;
uint16_t offset;
uint32_t xattr_idx;
} sqfs_inode_dir_ext_t;
typedef struct {
uint32_t count;
uint32_t start_block;
uint32_t inode_number;
} sqfs_dir_header_t;
typedef struct {
uint16_t offset;
uint16_t inode_number;
uint16_t type;
uint16_t size;
uint8_t name[];
} sqfs_dir_entry_t;
typedef enum {
SQFS_COMP_GZIP = 1,
SQFS_COMP_LZMA = 2,
SQFS_COMP_LZO = 3,
SQFS_COMP_XZ = 4,
SQFS_COMP_LZ4 = 5,
SQFS_COMP_ZSTD = 6,
} E_SQFS_COMPRESSOR;
typedef enum {
SQFS_FLAG_UNCOMPRESSED_INODES = 0x0001,
SQFS_FLAG_UNCOMPRESSED_DATA = 0x0002,
SQFS_FLAG_UNCOMPRESSED_FRAGMENTS = 0x0008,
SQFS_FLAG_NO_FRAGMENTS = 0x0010,
SQFS_FLAG_ALWAYS_FRAGMENTS = 0x0020,
SQFS_FLAG_DUPLICATES = 0x0040,
SQFS_FLAG_EXPORTABLE = 0x0080,
SQFS_FLAG_UNCOMPRESSED_XATTRS = 0x0100,
SQFS_FLAG_NO_XATTRS = 0x0200,
SQFS_FLAG_COMPRESSOR_OPTIONS = 0x0400,
SQFS_FLAG_UNCOMPRESSED_IDS = 0x0800,
} E_SQFS_SUPER_FLAGS;
typedef enum {
SQFS_INODE_DIR = 1,
SQFS_INODE_FILE = 2,
SQFS_INODE_SLINK = 3,
SQFS_INODE_BDEV = 4,
SQFS_INODE_CDEV = 5,
SQFS_INODE_FIFO = 6,
SQFS_INODE_SOCKET = 7,
SQFS_INODE_EXT_DIR = 8,
SQFS_INODE_EXT_FILE = 9,
SQFS_INODE_EXT_SLINK = 10,
SQFS_INODE_EXT_BDEV = 11,
SQFS_INODE_EXT_CDEV = 12,
SQFS_INODE_EXT_FIFO = 13,
SQFS_INODE_EXT_SOCKET = 14,
} E_SQFS_INODE_TYPE;
#endif /* SQUASHFS_H */

99
sqfs/super.c Normal file
View file

@ -0,0 +1,99 @@
/* SPDX-License-Identifier: ISC */
#include <endian.h>
#include <string.h>
#include <stdio.h>
#include "util/util.h"
#include "pkg2sqfs.h"
int sqfs_super_init(sqfs_super_t *s, int64_t timestamp,
uint32_t blocksize, E_SQFS_COMPRESSOR comp)
{
memset(s, 0, sizeof(*s));
if (blocksize & (blocksize - 1)) {
fputs("Block size must be a power of 2\n", stderr);
return -1;
}
if (blocksize < 4096 || blocksize >= (1 << 24)) {
fputs("Block size must be between 4k and 1M\n", stderr);
return -1;
}
if (timestamp < 0 || timestamp > 0x00000000FFFFFFFF) {
fputs("Timestamp too large for squashfs\n", stderr);
return -1;
}
s->magic = SQFS_MAGIC;
s->modification_time = timestamp;
s->block_size = blocksize;
s->compression_id = comp;
s->flags = SQFS_FLAG_NO_FRAGMENTS | SQFS_FLAG_NO_XATTRS;
s->version_major = SQFS_VERSION_MAJOR;
s->version_minor = SQFS_VERSION_MINOR;
s->bytes_used = sizeof(*s);
s->id_table_start = 0xFFFFFFFFFFFFFFFFUL;
s->xattr_id_table_start = 0xFFFFFFFFFFFFFFFFUL;
s->inode_table_start = 0xFFFFFFFFFFFFFFFFUL;
s->directory_table_start = 0xFFFFFFFFFFFFFFFFUL;
s->fragment_table_start = 0xFFFFFFFFFFFFFFFFUL;
s->export_table_start = 0xFFFFFFFFFFFFFFFFUL;
while (blocksize != 0) {
s->block_log += 1;
blocksize >>= 1;
}
s->block_log -= 1;
return 0;
}
int sqfs_super_write(sqfs_info_t *info)
{
sqfs_super_t copy;
ssize_t ret;
off_t off;
copy.magic = htole32(info->super.magic);
copy.inode_count = htole32(info->super.inode_count);
copy.modification_time = htole32(info->super.modification_time);
copy.block_size = htole32(info->super.block_size);
copy.fragment_entry_count = htole32(info->super.fragment_entry_count);
copy.compression_id = htole16(info->super.compression_id);
copy.block_log = htole16(info->super.block_log);
copy.flags = htole16(info->super.flags);
copy.id_count = htole16(info->super.id_count);
copy.version_major = htole16(info->super.version_major);
copy.version_minor = htole16(info->super.version_minor);
copy.root_inode_ref = htole64(info->super.root_inode_ref);
copy.bytes_used = htole64(info->super.bytes_used);
copy.id_table_start = htole64(info->super.id_table_start);
copy.xattr_id_table_start = htole64(info->super.xattr_id_table_start);
copy.inode_table_start = htole64(info->super.inode_table_start);
copy.directory_table_start = htole64(info->super.directory_table_start);
copy.fragment_table_start = htole64(info->super.fragment_table_start);
copy.export_table_start = htole64(info->super.export_table_start);
off = lseek(info->outfd, 0, SEEK_SET);
if (off == (off_t)-1) {
perror("seek on output file");
return -1;
}
ret = write_retry(info->outfd, &copy, sizeof(copy));
if (ret < 0) {
perror("Error writing squashfs super block");
return -1;
}
if (ret == 0) {
fputs("Truncated write trying to write squashfs super block\n",
stderr);
return -1;
}
return 0;
}

58
sqfs/table.c Normal file
View file

@ -0,0 +1,58 @@
/* SPDX-License-Identifier: ISC */
#include "pkg2sqfs.h"
int sqfs_write_table(sqfs_info_t *info, const void *data, size_t entsize,
size_t count, uint64_t *startblock)
{
size_t ent_per_blocks = SQFS_META_BLOCK_SIZE / entsize;
uint64_t blocks[count / ent_per_blocks + 1];
size_t i, blkidx = 0, tblsize;
meta_writer_t *m;
ssize_t ret;
/* Write actual data. Whenever we cross a block boundary, remember
the block start offset */
m = meta_writer_create(info->outfd);
if (m == NULL)
return -1;
for (i = 0; i < count; ++i) {
if (blkidx == 0 || m->written > blocks[blkidx - 1])
blocks[blkidx++] = m->written;
if (meta_writer_append(m, data, entsize))
goto fail;
data = (const char *)data + entsize;
}
if (meta_writer_flush(m))
goto fail;
for (i = 0; i < blkidx; ++i)
blocks[i] = htole64(blocks[i] + info->super.bytes_used);
info->super.bytes_used += m->written;
meta_writer_destroy(m);
/* write new index table */
*startblock = info->super.bytes_used;
tblsize = sizeof(blocks[0]) * blkidx;
ret = write_retry(info->outfd, blocks, tblsize);
if (ret < 0) {
perror("writing index table");
return -1;
}
if ((size_t)ret < tblsize) {
fputs("index table truncated\n", stderr);
return -1;
}
info->super.bytes_used += tblsize;
return 0;
fail:
meta_writer_destroy(m);
return -1;
}

322
sqfs/vfs.c Normal file
View file

@ -0,0 +1,322 @@
/* SPDX-License-Identifier: ISC */
#include <sys/stat.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include "pkg2sqfs.h"
static int id_to_index(vfs_t *fs, uint32_t id, uint16_t *out)
{
size_t i, new_sz;
void *new;
for (i = 0; i < fs->num_ids; ++i) {
if (fs->id_tbl[i] == id) {
*out = i;
return 0;
}
}
if (fs->num_ids > 0xFFFF) {
fputs("too many unique UIDs/GIDs (more than 64k)!\n", stderr);
return -1;
}
if (fs->num_ids == fs->max_ids) {
new_sz = fs->max_ids + 10;
new = realloc(fs->id_tbl, new_sz * sizeof(fs->id_tbl[0]));
if (new == NULL) {
perror("expanding ID table");
return -1;
}
fs->max_ids = new_sz;
fs->id_tbl = new;
}
*out = fs->num_ids;
fs->id_tbl[fs->num_ids++] = id;
return 0;
}
static node_t *node_create(vfs_t *fs, node_t *parent, size_t extra,
const char *name, size_t name_len,
uint16_t mode, uint32_t uid, uint32_t gid)
{
node_t *n = calloc(1, sizeof(*n) + extra + name_len + 1);
if (n == NULL) {
perror("allocating vfs node");
return NULL;
}
if (parent != NULL) {
n->parent = parent;
n->next = parent->data.dir->children;
parent->data.dir->children = n;
}
if (id_to_index(fs, uid, &n->uid_idx))
return NULL;
if (id_to_index(fs, gid, &n->gid_idx))
return NULL;
n->mode = mode;
n->name = (char *)n->extra;
memcpy(n->extra, name, name_len);
return n;
}
static node_t *node_from_ent(vfs_t *fs, sqfs_super_t *s, image_entry_t *ent)
{
node_t *n, *parent = fs->root;
const char *path = ent->name;
size_t len, extra;
char *end;
next:
end = strchrnul(path, '/');
len = end - path;
for (n = parent->data.dir->children; n != NULL; n = n->next) {
if (!strncmp(n->name, path, len) && strlen(n->name) == len) {
if (path[len] != '/')
goto fail_exists;
if (!S_ISDIR(n->mode))
goto fail_not_dir;
parent = n;
path += len + 1;
goto next;
}
}
if (path[len] == '/') {
if (S_ISDIR(ent->mode)) {
n = node_create(fs, parent, sizeof(dir_info_t),
path, len, S_IFDIR | fs->default_mode,
fs->default_uid, fs->default_gid);
if (n == NULL)
return NULL;
n->data.dir =
(dir_info_t *)(n->name + strlen(n->name) + 1);
parent = n;
path += len + 1;
goto next;
}
goto fail_no_ent;
}
switch (ent->mode & S_IFMT) {
case S_IFREG:
extra = sizeof(file_info_t);
extra += (ent->data.file.size / s->block_size) *
sizeof(uint32_t);
break;
case S_IFLNK:
extra = strlen(ent->data.symlink.target) + 1;
break;
case S_IFDIR:
extra = sizeof(dir_info_t);
break;
default:
extra = 0;
break;
}
n = node_create(fs, parent, extra, path, len,
ent->mode, ent->uid, ent->gid);
if (n == NULL)
return NULL;
switch (ent->mode & S_IFMT) {
case S_IFREG:
n->data.file = (file_info_t *)(n->name + strlen(n->name) + 1);
n->data.file->size = ent->data.file.size;
n->data.file->id = ent->data.file.id;
n->data.file->startblock = 0xFFFFFFFFFFFFFFFFUL;
n->data.file->fragment = 0xFFFFFFFFL;
break;
case S_IFLNK:
n->data.symlink = n->name + strlen(n->name) + 1;
strcpy((char *)n->data.symlink, ent->data.symlink.target);
break;
case S_IFBLK:
case S_IFCHR:
n->data.device = ent->data.device.devno;
break;
case S_IFDIR:
n->data.dir = (dir_info_t *)(n->name + strlen(n->name) + 1);
break;
default:
break;
}
return n;
fail_no_ent:
fprintf(stderr, "cannot create /%s: '%.*s' does not exist\n",
ent->name, (int)len, path);
return NULL;
fail_exists:
fprintf(stderr, "cannot create /%s: already exists\n", ent->name);
return NULL;
fail_not_dir:
fprintf(stderr, "cannot create /%s: %.*s is not a directory\n",
ent->name, (int)len, path);
return NULL;
}
static void node_recursive_delete(node_t *n)
{
node_t *child;
if (n != NULL && S_ISDIR(n->mode)) {
while (n->data.dir->children != NULL) {
child = n->data.dir->children;
n->data.dir->children = child->next;
node_recursive_delete(child);
}
}
free(n);
}
typedef int(*node_cb)(vfs_t *fs, node_t *n, void *user);
static int foreach_node(vfs_t *fs, node_t *root, void *user, node_cb cb)
{
bool has_subdirs = false;
node_t *it;
for (it = root->data.dir->children; it != NULL; it = it->next) {
if (S_ISDIR(it->mode)) {
has_subdirs = true;
break;
}
}
if (has_subdirs) {
for (it = root->data.dir->children; it != NULL; it = it->next) {
if (!S_ISDIR(it->mode))
continue;
if (foreach_node(fs, it, user, cb) != 0)
return -1;
}
}
for (it = root->data.dir->children; it != NULL; it = it->next) {
if (cb(fs, it, user))
return -1;
}
if (root->parent == NULL) {
if (cb(fs, root, user))
return -1;
}
return 0;
}
static int alloc_inum(vfs_t *fs, node_t *n, void *user)
{
uint32_t *counter = user;
(void)fs;
if (*counter == 0xFFFFFFFF) {
fputs("too many inodes (more than 2^32 - 2)!\n", stderr);
return -1;
}
n->inode_num = (*counter)++;
return 0;
}
static int link_node_to_tbl(vfs_t *fs, node_t *n, void *user)
{
(void)user;
fs->node_tbl[n->inode_num] = n;
return 0;
}
int create_vfs_tree(sqfs_info_t *info)
{
image_entry_t *ent, *list;
uint32_t counter = 2;
vfs_t *fs = &info->fs;
node_t *n;
size_t i;
if (image_entry_list_from_package(info->rd, &list))
return -1;
fs->root = node_create(fs, NULL, sizeof(dir_info_t),
"", 0, S_IFDIR | 0755, 0, 0);
if (fs->root == NULL)
goto fail;
fs->root->data.dir = (dir_info_t *)(fs->root->name + strlen(fs->root->name) + 1);
for (ent = list; ent != NULL; ent = ent->next) {
n = node_from_ent(fs, &info->super, ent);
if (n == NULL)
goto fail;
}
if (foreach_node(fs, fs->root, &counter, alloc_inum))
goto fail;
fs->num_inodes = counter;
fs->node_tbl = calloc(fs->num_inodes, sizeof(fs->node_tbl[0]));
if (fs->node_tbl == NULL)
goto fail_oom;
if (foreach_node(fs, fs->root, NULL, link_node_to_tbl))
goto fail;
info->super.inode_count = fs->num_inodes - 2;
info->super.id_count = fs->num_ids;
image_entry_free_list(list);
for (i = 0; i < fs->num_ids; ++i)
fs->id_tbl[i] = htole32(fs->id_tbl[i]);
return 0;
fail_oom:
fputs("out of memory\n", stderr);
fail:
image_entry_free_list(list);
free(fs->node_tbl);
free(fs->id_tbl);
node_recursive_delete(fs->root);
return -1;
}
void destroy_vfs_tree(vfs_t *fs)
{
free(fs->node_tbl);
free(fs->id_tbl);
node_recursive_delete(fs->root);
}
node_t *vfs_node_from_file_id(vfs_t *fs, uint32_t id)
{
size_t i;
for (i = 0; i < fs->num_inodes; ++i) {
if (fs->node_tbl[i] == NULL)
continue;
if (!S_ISREG(fs->node_tbl[i]->mode))
continue;
if (fs->node_tbl[i]->data.file->id == id)
return fs->node_tbl[i];
}
return NULL;
}