pygos/pkg-utils
1
0
Fork 0
mirror of https://github.com/pygos/pkg-utils.git synced 2024-10-06 08:19:59 +02:00
Code Issues Releases Wiki Activity

Remove pkg2sqfs

Browse source 
Rewritten and moved to a seperate project.

Signed-off-by: David Oberhollenzer <goliath@infraroot.at>
This commit is contained in:
David Oberhollenzer 2019-06-09 02:18:27 +02:00
parent e08c819606
commit f426e4ef91
12 changed files with 1 additions and 1869 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Makefile.am
View file

@ -10,4 +10,3 @@ EXTRA_DIST = autogen.sh LICENSE README.md
include lib/Makemodule.am
include main/Makemodule.am
include sqfs/Makemodule.am

198
include/sqfs/squashfs.h
View file

@ -1,198 +0,0 @@
/* SPDX-License-Identifier: ISC */
#ifndef SQUASHFS_H
#define SQUASHFS_H
#include "comp/compressor.h"
#include "pkg/pkgformat.h"
#include <stdint.h>
#define SQFS_MAGIC 0x73717368
#define SQFS_VERSION_MAJOR 4
#define SQFS_VERSION_MINOR 0
#define SQFS_META_BLOCK_SIZE 8192
#define SQFS_DEFAULT_BLOCK_SIZE 131072
#define SQFS_DEVBLK_SIZE 4096
#define SQFS_MAX_DIR_ENT 256
typedef struct {
uint32_t magic;
uint32_t inode_count;
uint32_t modification_time;
uint32_t block_size;
uint32_t fragment_entry_count;
uint16_t compression_id;
uint16_t block_log;
uint16_t flags;
uint16_t id_count;
uint16_t version_major;
uint16_t version_minor;
uint64_t root_inode_ref;
uint64_t bytes_used;
uint64_t id_table_start;
uint64_t xattr_id_table_start;
uint64_t inode_table_start;
uint64_t directory_table_start;
uint64_t fragment_table_start;
uint64_t export_table_start;
} __attribute__((packed)) sqfs_super_t;
typedef struct {
uint64_t start_offset;
uint32_t size;
uint32_t pad0;
} sqfs_fragment_t;
typedef struct {
uint16_t type;
uint16_t mode;
uint16_t uid_idx;
uint16_t gid_idx;
uint32_t mod_time;
uint32_t inode_number;
} sqfs_inode_t;
typedef struct {
uint32_t nlink;
uint32_t devno;
} sqfs_inode_dev_t;
typedef struct {
uint32_t nlink;
uint32_t devno;
uint32_t xattr_idx;
} sqfs_inode_dev_ext_t;
typedef struct {
uint32_t nlink;
uint32_t target_size;
uint8_t target[];
} sqfs_inode_slink_t;
typedef struct {
uint32_t blocks_start;
uint32_t fragment_index;
uint32_t fragment_offset;
uint32_t file_size;
uint32_t block_sizes[];
} sqfs_inode_file_t;
typedef struct {
uint64_t blocks_start;
uint64_t file_size;
uint64_t sparse;
uint32_t nlink;
uint32_t fragment_idx;
uint32_t fragment_offset;
uint32_t xattr_idx;
uint32_t block_sizes[];
} sqfs_inode_file_ext_t;
typedef struct {
uint32_t start_block;
uint32_t nlink;
uint16_t size;
uint16_t offset;
uint32_t parent_inode;
} sqfs_inode_dir_t;
typedef struct {
uint32_t nlink;
uint32_t size;
uint32_t start_block;
uint32_t parent_inode;
uint16_t inodex_count;
uint16_t offset;
uint32_t xattr_idx;
} sqfs_inode_dir_ext_t;
typedef struct {
uint32_t count;
uint32_t start_block;
uint32_t inode_number;
} sqfs_dir_header_t;
typedef struct {
uint16_t offset;
uint16_t inode_number;
uint16_t type;
uint16_t size;
uint8_t name[];
} sqfs_dir_entry_t;
typedef enum {
SQFS_COMP_GZIP = 1,
SQFS_COMP_LZMA = 2,
SQFS_COMP_LZO = 3,
SQFS_COMP_XZ = 4,
SQFS_COMP_LZ4 = 5,
SQFS_COMP_ZSTD = 6,
SQFS_COMP_MIN = 1,
SQFS_COMP_MAX = 6,
} E_SQFS_COMPRESSOR;
typedef enum {
SQFS_FLAG_UNCOMPRESSED_INODES = 0x0001,
SQFS_FLAG_UNCOMPRESSED_DATA = 0x0002,
SQFS_FLAG_UNCOMPRESSED_FRAGMENTS = 0x0008,
SQFS_FLAG_NO_FRAGMENTS = 0x0010,
SQFS_FLAG_ALWAYS_FRAGMENTS = 0x0020,
SQFS_FLAG_DUPLICATES = 0x0040,
SQFS_FLAG_EXPORTABLE = 0x0080,
SQFS_FLAG_UNCOMPRESSED_XATTRS = 0x0100,
SQFS_FLAG_NO_XATTRS = 0x0200,
SQFS_FLAG_COMPRESSOR_OPTIONS = 0x0400,
SQFS_FLAG_UNCOMPRESSED_IDS = 0x0800,
} E_SQFS_SUPER_FLAGS;
typedef enum {
SQFS_INODE_DIR = 1,
SQFS_INODE_FILE = 2,
SQFS_INODE_SLINK = 3,
SQFS_INODE_BDEV = 4,
SQFS_INODE_CDEV = 5,
SQFS_INODE_FIFO = 6,
SQFS_INODE_SOCKET = 7,
SQFS_INODE_EXT_DIR = 8,
SQFS_INODE_EXT_FILE = 9,
SQFS_INODE_EXT_SLINK = 10,
SQFS_INODE_EXT_BDEV = 11,
SQFS_INODE_EXT_CDEV = 12,
SQFS_INODE_EXT_FIFO = 13,
SQFS_INODE_EXT_SOCKET = 14,
} E_SQFS_INODE_TYPE;
typedef struct {
uint8_t data[SQFS_META_BLOCK_SIZE + 2];
uint8_t scratch[SQFS_META_BLOCK_SIZE];
size_t offset;
size_t written;
int outfd;
compressor_stream_t *strm;
} meta_writer_t;
int sqfs_super_init(sqfs_super_t *s, int64_t timestamp,
uint32_t blocksize, E_SQFS_COMPRESSOR comp);
int sqfs_super_write(const sqfs_super_t *super, int outfd);
compressor_stream_t *sqfs_get_compressor(sqfs_super_t *s);
meta_writer_t *meta_writer_create(int fd, compressor_stream_t *strm);
void meta_writer_destroy(meta_writer_t *m);
int meta_writer_flush(meta_writer_t *m);
int meta_writer_append(meta_writer_t *m, const void *data, size_t size);
int sqfs_write_ids(int outfd, sqfs_super_t *super,
uint32_t *id_tbl, size_t count,
compressor_stream_t *strm);
int sqfs_write_fragment_table(int outfd, sqfs_super_t *super,
sqfs_fragment_t *fragments, size_t count,
compressor_stream_t *strm);
#endif /* SQUASHFS_H */

5
lib/Makemodule.am
View file

@ -33,7 +33,4 @@ libpkg_a_SOURCES += include/pkg/pkgio.h include/pkg/pkgwriter.h
libpkg_a_SOURCES += lib/pkg/pkgreader.c lib/pkg/pkgwriter.c
libpkg_a_SOURCES += lib/pkg/pkg_unpack.c lib/pkg/pkgio_rd_image_entry.c
libsqfs_a_SOURCES = include/sqfs/squashfs.h lib/sqfs/super.c
libsqfs_a_SOURCES += lib/sqfs/meta_writer.c lib/sqfs/table.c
noinst_LIBRARIES += libutil.a libfilelist.a libcomp.a libpkg.a libsqfs.a
noinst_LIBRARIES += libutil.a libfilelist.a libcomp.a libpkg.a

92
lib/sqfs/meta_writer.c
View file

@ -1,92 +0,0 @@
/* SPDX-License-Identifier: ISC */
#include "sqfs/squashfs.h"
#include "util/util.h"
#include <string.h>
#include <stdlib.h>
#include <stdio.h>
meta_writer_t *meta_writer_create(int fd, compressor_stream_t *strm)
{
meta_writer_t *m = calloc(1, sizeof(*m));
if (m == NULL) {
perror("creating meta data writer");
return NULL;
}
m->strm = strm;
m->outfd = fd;
return m;
}
void meta_writer_destroy(meta_writer_t *m)
{
free(m);
}
int meta_writer_flush(meta_writer_t *m)
{
ssize_t ret, count;
if (m->offset == 0)
return 0;
ret = m->strm->do_block(m->strm, m->data + 2, m->scratch,
m->offset, sizeof(m->scratch));
if (ret < 0)
return -1;
if (ret > 0 && (size_t)ret < m->offset) {
memcpy(m->data + 2, m->scratch, ret);
((uint16_t *)m->data)[0] = htole16(ret);
count = ret + 2;
} else {
((uint16_t *)m->data)[0] = htole16(m->offset | 0x8000);
count = m->offset + 2;
}
ret = write_retry(m->outfd, m->data, count);
if (ret < 0) {
perror("writing meta data");
return -1;
}
if (ret < count) {
fputs("meta data was truncated\n", stderr);
return -1;
}
memset(m->data, 0, sizeof(m->data));
m->offset = 0;
m->written += count;
return 0;
}
int meta_writer_append(meta_writer_t *m, const void *data, size_t size)
{
size_t diff;
while (size != 0) {
diff = sizeof(m->data) - 2 - m->offset;
if (diff == 0) {
if (meta_writer_flush(m))
return -1;
diff = sizeof(m->data) - 2;
}
if (diff > size)
diff = size;
memcpy(m->data + 2 + m->offset, data, diff);
m->offset += diff;
size -= diff;
data = (const char *)data + diff;
}
if (m->offset == (sizeof(m->data) - 2))
return meta_writer_flush(m);
return 0;
}

129
lib/sqfs/super.c
View file

@ -1,129 +0,0 @@
/* SPDX-License-Identifier: ISC */
#include <endian.h>
#include <string.h>
#include <unistd.h>
#include <stdio.h>
#include "sqfs/squashfs.h"
#include "util/util.h"
int sqfs_super_init(sqfs_super_t *s, int64_t timestamp,
uint32_t blocksize, E_SQFS_COMPRESSOR comp)
{
memset(s, 0, sizeof(*s));
if (blocksize & (blocksize - 1)) {
fputs("Block size must be a power of 2\n", stderr);
return -1;
}
if (blocksize < 4096 || blocksize >= (1 << 24)) {
fputs("Block size must be between 4k and 1M\n", stderr);
return -1;
}
if (timestamp < 0 || timestamp > 0x00000000FFFFFFFF) {
fputs("Timestamp too large for squashfs\n", stderr);
return -1;
}
s->magic = SQFS_MAGIC;
s->modification_time = timestamp;
s->block_size = blocksize;
s->compression_id = comp;
s->flags = SQFS_FLAG_NO_FRAGMENTS | SQFS_FLAG_NO_XATTRS;
s->version_major = SQFS_VERSION_MAJOR;
s->version_minor = SQFS_VERSION_MINOR;
s->bytes_used = sizeof(*s);
s->id_table_start = 0xFFFFFFFFFFFFFFFFUL;
s->xattr_id_table_start = 0xFFFFFFFFFFFFFFFFUL;
s->inode_table_start = 0xFFFFFFFFFFFFFFFFUL;
s->directory_table_start = 0xFFFFFFFFFFFFFFFFUL;
s->fragment_table_start = 0xFFFFFFFFFFFFFFFFUL;
s->export_table_start = 0xFFFFFFFFFFFFFFFFUL;
while (blocksize != 0) {
s->block_log += 1;
blocksize >>= 1;
}
s->block_log -= 1;
return 0;
}
int sqfs_super_write(const sqfs_super_t *super, int outfd)
{
sqfs_super_t copy;
ssize_t ret;
off_t off;
copy.magic = htole32(super->magic);
copy.inode_count = htole32(super->inode_count);
copy.modification_time = htole32(super->modification_time);
copy.block_size = htole32(super->block_size);
copy.fragment_entry_count = htole32(super->fragment_entry_count);
copy.compression_id = htole16(super->compression_id);
copy.block_log = htole16(super->block_log);
copy.flags = htole16(super->flags);
copy.id_count = htole16(super->id_count);
copy.version_major = htole16(super->version_major);
copy.version_minor = htole16(super->version_minor);
copy.root_inode_ref = htole64(super->root_inode_ref);
copy.bytes_used = htole64(super->bytes_used);
copy.id_table_start = htole64(super->id_table_start);
copy.xattr_id_table_start = htole64(super->xattr_id_table_start);
copy.inode_table_start = htole64(super->inode_table_start);
copy.directory_table_start = htole64(super->directory_table_start);
copy.fragment_table_start = htole64(super->fragment_table_start);
copy.export_table_start = htole64(super->export_table_start);
off = lseek(outfd, 0, SEEK_SET);
if (off == (off_t)-1) {
perror("squashfs writing super block: seek on output file");
return -1;
}
ret = write_retry(outfd, &copy, sizeof(copy));
if (ret < 0) {
perror("squashfs writing super block");
return -1;
}
if (ret == 0) {
fputs("squashfs writing super block: truncated write\n",
stderr);
return -1;
}
return 0;
}
compressor_stream_t *sqfs_get_compressor(sqfs_super_t *s)
{
PKG_COMPRESSION id;
compressor_t *cmp;
size_t xz_dictsz;
void *options;
switch (s->compression_id) {
case SQFS_COMP_GZIP:
id = PKG_COMPRESSION_ZLIB;
options = NULL;
break;
case SQFS_COMP_XZ:
id = PKG_COMPRESSION_LZMA;
xz_dictsz = s->block_size;
options = &xz_dictsz;
break;
default:
fputs("unsupported compressor\n", stderr);
return NULL;
}
cmp = compressor_by_id(id);
if (cmp == NULL)
return NULL;
return cmp->compression_stream(cmp, options);
}

92
lib/sqfs/table.c
View file

@ -1,92 +0,0 @@
/* SPDX-License-Identifier: ISC */
#include "sqfs/squashfs.h"
#include "util/util.h"
#include <endian.h>
#include <stdio.h>
static int sqfs_write_table(int outfd, sqfs_super_t *super, const void *data,
size_t entsize, size_t count, uint64_t *startblock,
compressor_stream_t *strm)
{
size_t ent_per_blocks = SQFS_META_BLOCK_SIZE / entsize;
uint64_t blocks[count / ent_per_blocks + 1];
size_t i, blkidx = 0, tblsize;
meta_writer_t *m;
ssize_t ret;
/* Write actual data. Whenever we cross a block boundary, remember
the block start offset */
m = meta_writer_create(outfd, strm);
if (m == NULL)
return -1;
for (i = 0; i < count; ++i) {
if (blkidx == 0 || m->written > blocks[blkidx - 1])
blocks[blkidx++] = m->written;
if (meta_writer_append(m, data, entsize))
goto fail;
data = (const char *)data + entsize;
}
if (meta_writer_flush(m))
goto fail;
for (i = 0; i < blkidx; ++i)
blocks[i] = htole64(blocks[i] + super->bytes_used);
super->bytes_used += m->written;
meta_writer_destroy(m);
/* write new index table */
*startblock = super->bytes_used;
tblsize = sizeof(blocks[0]) * blkidx;
ret = write_retry(outfd, blocks, tblsize);
if (ret < 0) {
perror("writing index table");
return -1;
}
if ((size_t)ret < tblsize) {
fputs("index table truncated\n", stderr);
return -1;
}
super->bytes_used += tblsize;
return 0;
fail:
meta_writer_destroy(m);
return -1;
}
int sqfs_write_fragment_table(int outfd, sqfs_super_t *super,
sqfs_fragment_t *fragments, size_t count,
compressor_stream_t *strm)
{
super->fragment_entry_count = count;
return sqfs_write_table(outfd, super, fragments, sizeof(fragments[0]),
count, &super->fragment_table_start, strm);
}
int sqfs_write_ids(int outfd, sqfs_super_t *super,
uint32_t *id_tbl, size_t count,
compressor_stream_t *strm)
{
size_t i;
int ret;
for (i = 0; i < count; ++i)
id_tbl[i] = htole32(id_tbl[i]);
ret = sqfs_write_table(outfd, super, id_tbl, sizeof(id_tbl[0]),
count, &super->id_table_start, strm);
for (i = 0; i < count; ++i)
id_tbl[i] = htole32(id_tbl[i]);
return ret;
}

16
sqfs/Makemodule.am
View file

@ -1,16 +0,0 @@
pkg2sqfs_SOURCES = sqfs/pkg2sqfs.c
pkg2sqfs_SOURCES += sqfs/vfs.c sqfs/pkg2sqfs.h sqfs/block.c
pkg2sqfs_SOURCES += sqfs/meta.c
pkg2sqfs_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/sqfs
pkg2sqfs_CFLAGS = $(AM_CFLAGS)
pkg2sqfs_LDADD = libsqfs.a libpkg.a libutil.a libfilelist.a libcomp.a
if WITH_LZMA
pkg2sqfs_LDADD += $(XZ_LIBS)
endif
if WITH_ZLIB
pkg2sqfs_LDADD += $(ZLIB_LIBS)
endif
bin_PROGRAMS += pkg2sqfs

225
sqfs/block.c
View file

@ -1,225 +0,0 @@
/* SPDX-License-Identifier: ISC */
#include "pkg2sqfs.h"
static int write_block(file_info_t *file, sqfs_info_t *info,
compressor_stream_t *strm)
{
size_t idx, bs;
ssize_t ret;
void *ptr;
idx = info->file_block_count++;
bs = info->super.block_size;
ret = strm->do_block(strm, info->block, info->scratch, bs, bs);
if (ret < 0)
return -1;
if (ret > 0) {
ptr = info->scratch;
bs = ret;
file->blocksizes[idx] = bs;
} else {
ptr = info->block;
file->blocksizes[idx] = bs | (1 << 24);
}
ret = write_retry(info->outfd, ptr, bs);
if (ret < 0) {
perror("writing to output file");
return -1;
}
if ((size_t)ret < bs) {
fputs("write to output file truncated\n", stderr);
return -1;
}
info->super.bytes_used += bs;
return 0;
}
static int flush_fragments(sqfs_info_t *info, compressor_stream_t *strm)
{
size_t newsz, size;
file_info_t *fi;
uint64_t offset;
void *new, *ptr;
ssize_t ret;
if (info->num_fragments == info->max_fragments) {
newsz = info->max_fragments ? info->max_fragments * 2 : 16;
new = realloc(info->fragments,
sizeof(info->fragments[0]) * newsz);
if (new == NULL) {
perror("appending to fragment table");
return -1;
}
info->max_fragments = newsz;
info->fragments = new;
}
offset = info->super.bytes_used;
size = info->frag_offset;
for (fi = info->frag_list; fi != NULL; fi = fi->frag_next)
fi->fragment = info->num_fragments;
ret = strm->do_block(strm, info->fragment, info->scratch,
size, info->super.block_size);
if (ret < 0)
return -1;
info->fragments[info->num_fragments].start_offset = htole64(offset);
info->fragments[info->num_fragments].pad0 = 0;
if (ret > 0 && (size_t)ret < size) {
size = ret;
info->fragments[info->num_fragments].size = htole32(size);
ptr = info->scratch;
} else {
info->fragments[info->num_fragments].size =
htole32(size | (1 << 24));
ptr = info->fragment;
}
info->num_fragments += 1;
ret = write_retry(info->outfd, ptr, size);
if (ret < 0) {
perror("writing to output file");
return -1;
}
if ((size_t)ret < size) {
fputs("write to output file truncated\n", stderr);
return -1;
}
memset(info->fragment, 0, info->super.block_size);
info->super.bytes_used += size;
info->frag_offset = 0;
info->frag_list = NULL;
info->super.flags &= ~SQFS_FLAG_NO_FRAGMENTS;
info->super.flags |= SQFS_FLAG_ALWAYS_FRAGMENTS;
return 0;
}
static int add_fragment(file_info_t *fi, sqfs_info_t *info, size_t size,
compressor_stream_t *strm)
{
if (info->frag_offset + size > info->super.block_size) {
if (flush_fragments(info, strm))
return -1;
}
fi->fragment_offset = info->frag_offset;
fi->frag_next = info->frag_list;
info->frag_list = fi;
memcpy((char *)info->fragment + info->frag_offset, info->block, size);
info->frag_offset += size;
return 0;
}
static int process_file(node_t *node, sqfs_info_t *info,
compressor_stream_t *strm)
{
uint64_t count = node->data.file->size;
int ret;
size_t diff;
node->data.file->startblock = info->super.bytes_used;
info->file_block_count = 0;
while (count != 0) {
diff = count > (uint64_t)info->super.block_size ?
info->super.block_size : count;
ret = pkg_reader_read_payload(info->rd, info->block, diff);
if (ret < 0)
return -1;
if ((size_t)ret < diff)
goto fail_trunc;
if (diff < info->super.block_size) {
if (add_fragment(node->data.file, info, diff, strm))
return -1;
} else {
if (write_block(node->data.file, info, strm))
return -1;
}
count -= diff;
}
return 0;
fail_trunc:
fprintf(stderr, "%s: truncated data block\n",
pkg_reader_get_filename(info->rd));
return -1;
}
int pkg_data_to_sqfs(sqfs_info_t *info, compressor_stream_t *strm)
{
file_data_t frec;
record_t *hdr;
node_t *node;
int ret;
if (pkg_reader_rewind(info->rd))
return -1;
memset(info->fragment, 0, info->super.block_size);
for (;;) {
ret = pkg_reader_get_next_record(info->rd);
if (ret == 0)
break;
if (ret < 0)
return -1;
hdr = pkg_reader_current_record_header(info->rd);
if (hdr->magic != PKG_MAGIC_DATA)
continue;
for (;;) {
ret = pkg_reader_read_payload(info->rd, &frec,
sizeof(frec));
if (ret == 0)
break;
if (ret < 0)
return -1;
if ((size_t)ret < sizeof(frec))
goto fail_trunc;
frec.id = le32toh(frec.id);
node = vfs_node_from_file_id(&info->fs, frec.id);
if (node == NULL)
goto fail_meta;
if (process_file(node, info, strm))
return -1;
}
}
if (info->frag_offset != 0) {
if (flush_fragments(info, strm))
return -1;
}
return 0;
fail_meta:
fprintf(stderr, "%s: missing meta information for file %u\n",
pkg_reader_get_filename(info->rd), (unsigned int)frec.id);
return -1;
fail_trunc:
fprintf(stderr, "%s: truncated file data record\n",
pkg_reader_get_filename(info->rd));
return -1;
}

297
sqfs/meta.c
View file

@ -1,297 +0,0 @@
/* SPDX-License-Identifier: ISC */
#include "pkg2sqfs.h"
static size_t hard_link_count(node_t *node)
{
size_t count;
node_t *n;
if (S_ISDIR(node->mode)) {
count = 2;
for (n = node->data.dir->children; n != NULL; n = n->next)
++count;
return count;
}
return 1;
}
static int write_dir(meta_writer_t *dm, dir_info_t *dir)
{
sqfs_dir_header_t hdr;
sqfs_dir_entry_t ent;
size_t i, count;
node_t *c, *d;
c = dir->children;
dir->size = 0;
dir->dref = (dm->written << 16) | dm->offset;
while (c != NULL) {
count = 0;
for (d = c; d != NULL; d = d->next) {
if ((d->inode_ref >> 16) != (c->inode_ref >> 16))
break;
if ((d->inode_num - c->inode_num) > 0xFFFF)
break;
count += 1;
}
if (count > SQFS_MAX_DIR_ENT)
count = SQFS_MAX_DIR_ENT;
hdr.count = htole32(count - 1);
hdr.start_block = htole32(c->inode_ref >> 16);
hdr.inode_number = htole32(c->inode_num);
dir->size += sizeof(hdr);
if (meta_writer_append(dm, &hdr, sizeof(hdr)))
return -1;
d = c;
for (i = 0; i < count; ++i) {
ent.offset = htole16(c->inode_ref & 0x0000FFFF);
ent.inode_number = htole16(c->inode_num - d->inode_num);
ent.type = htole16(c->type);
ent.size = htole16(strlen(c->name) - 1);
dir->size += sizeof(ent) + strlen(c->name);
if (meta_writer_append(dm, &ent, sizeof(ent)))
return -1;
if (meta_writer_append(dm, c->name, strlen(c->name)))
return -1;
c = c->next;
}
}
return 0;
}
static int write_inode(sqfs_info_t *info, meta_writer_t *im, meta_writer_t *dm,
node_t *node)
{
file_info_t *fi = NULL;
sqfs_inode_t base;
uint32_t bs;
uint64_t i;
node->inode_ref = (im->written << 16) | im->offset;
base.mode = htole16(node->mode);
base.uid_idx = htole16(node->uid_idx);
base.gid_idx = htole16(node->gid_idx);
base.mod_time = htole32(info->super.modification_time);
base.inode_number = htole32(node->inode_num);
switch (node->mode & S_IFMT) {
case S_IFLNK: node->type = SQFS_INODE_SLINK; break;
case S_IFBLK: node->type = SQFS_INODE_BDEV; break;
case S_IFCHR: node->type = SQFS_INODE_CDEV; break;
case S_IFDIR:
if (write_dir(dm, node->data.dir))
return -1;
node->type = SQFS_INODE_DIR;
i = node->data.dir->dref;
if ((i >> 16) > 0xFFFFFFFFUL || node->data.dir->size > 0xFFFF)
node->type = SQFS_INODE_EXT_DIR;
break;
case S_IFREG:
fi = node->data.file;
node->type = SQFS_INODE_FILE;
if (fi->startblock > 0xFFFFFFFFUL || fi->size > 0xFFFFFFFFUL ||
hard_link_count(node) > 1) {
node->type = SQFS_INODE_EXT_FILE;
}
break;
default:
assert(0);
}
base.type = htole16(node->type);
if (meta_writer_append(im, &base, sizeof(base)))
return -1;
switch (node->type) {
case SQFS_INODE_SLINK: {
sqfs_inode_slink_t slink = {
.nlink = htole32(hard_link_count(node)),
.target_size = htole32(strlen(node->data.symlink)),
};
if (meta_writer_append(im, &slink, sizeof(slink)))
return -1;
if (meta_writer_append(im, node->data.symlink,
le32toh(slink.target_size))) {
return -1;
}
break;
}
case SQFS_INODE_BDEV:
case SQFS_INODE_CDEV: {
sqfs_inode_dev_t dev = {
.nlink = htole32(hard_link_count(node)),
.devno = htole32(node->data.device),
};
return meta_writer_append(im, &dev, sizeof(dev));
}
case SQFS_INODE_EXT_FILE: {
sqfs_inode_file_ext_t ext = {
.blocks_start = htole64(fi->startblock),
.file_size = htole64(fi->size),
.sparse = htole64(0xFFFFFFFFFFFFFFFFUL),
.nlink = htole32(hard_link_count(node)),
.fragment_idx = htole32(fi->fragment),
.fragment_offset = htole32(fi->fragment_offset),
.xattr_idx = htole32(0xFFFFFFFF),
};
if (meta_writer_append(im, &ext, sizeof(ext)))
return -1;
goto out_file_blocks;
}
case SQFS_INODE_FILE: {
sqfs_inode_file_t reg = {
.blocks_start = htole32(fi->startblock),
.fragment_index = htole32(fi->fragment),
.fragment_offset = htole32(fi->fragment_offset),
.file_size = htole32(fi->size),
};
if (meta_writer_append(im, &reg, sizeof(reg)))
return -1;
goto out_file_blocks;
}
case SQFS_INODE_DIR: {
sqfs_inode_dir_t dir = {
.start_block = htole32(node->data.dir->dref >> 16),
.nlink = htole32(hard_link_count(node)),
.size = htole16(node->data.dir->size),
.offset = htole16(node->data.dir->dref & 0xFFFF),
.parent_inode = node->parent ?
htole32(node->parent->inode_num) : htole32(1),
};
return meta_writer_append(im, &dir, sizeof(dir));
}
case SQFS_INODE_EXT_DIR: {
sqfs_inode_dir_ext_t ext = {
.nlink = htole32(hard_link_count(node)),
.size = htole32(node->data.dir->size),
.start_block = htole32(node->data.dir->dref >> 16),
.parent_inode = node->parent ?
htole32(node->parent->inode_num) : htole32(1),
.inodex_count = htole32(0),
.offset = htole16(node->data.dir->dref & 0xFFFF),
.xattr_idx = htole32(0xFFFFFFFF),
};
return meta_writer_append(im, &ext, sizeof(ext));
}
default:
assert(0);
}
return 0;
out_file_blocks:
for (i = 0; i < fi->size / info->super.block_size; ++i) {
bs = htole32(fi->blocksizes[i]);
if (meta_writer_append(im, &bs, sizeof(bs)))
return -1;
}
return 0;
}
int sqfs_write_inodes(sqfs_info_t *info, compressor_stream_t *strm)
{
meta_writer_t *im, *dm;
size_t i, diff;
ssize_t ret;
FILE *tmp;
int tmpfd;
tmp = tmpfile();
if (tmp == NULL) {
perror("tmpfile");
return -1;
}
tmpfd = fileno(tmp);
im = meta_writer_create(info->outfd, strm);
if (im == NULL)
goto fail_tmp;
dm = meta_writer_create(tmpfd, strm);
if (dm == NULL)
goto fail_im;
for (i = 2; i < info->fs.num_inodes; ++i) {
if (write_inode(info, im, dm, info->fs.node_tbl[i]))
goto fail;
}
if (meta_writer_flush(im))
goto fail;
if (meta_writer_flush(dm))
goto fail;
info->super.root_inode_ref = info->fs.root->inode_ref;
info->super.inode_table_start = info->super.bytes_used;
info->super.bytes_used += im->written;
info->super.directory_table_start = info->super.bytes_used;
info->super.bytes_used += dm->written;
if (lseek(tmpfd, 0, SEEK_SET) == (off_t)-1) {
perror("rewind on directory temp file");
goto fail;
}
for (;;) {
ret = read_retry(tmpfd, dm->data, sizeof(dm->data));
if (ret < 0) {
perror("read from temp file");
goto fail;
}
if (ret == 0)
break;
diff = ret;
ret = write_retry(info->outfd, dm->data, diff);
if (ret < 0) {
perror("write to image file");
goto fail;
}
if ((size_t)ret < diff) {
fputs("copying meta data to image file: "
"truncated write\n", stderr);
goto fail;
}
}
meta_writer_destroy(dm);
meta_writer_destroy(im);
fclose(tmp);
return 0;
fail:
meta_writer_destroy(dm);
fail_im:
meta_writer_destroy(im);
fail_tmp:
fclose(tmp);
return -1;
}

316
sqfs/pkg2sqfs.c
View file

@ -1,316 +0,0 @@
/* SPDX-License-Identifier: ISC */
#include "pkg2sqfs.h"
static struct option long_opts[] = {
{ "timestamp", required_argument, NULL, 't' },
{ "compressor", required_argument, NULL, 'c' },
{ "block-size", required_argument, NULL, 'b' },
{ "dev-block-size", required_argument, NULL, 'B' },
{ "default-uid", required_argument, NULL, 'u' },
{ "default-gid", required_argument, NULL, 'g' },
{ "default-mode", required_argument, NULL, 'm' },
{ "force", no_argument, NULL, 'f' },
{ "version", no_argument, NULL, 'V' },
{ "help", no_argument, NULL, 'h' },
};
static const char *short_opts = "t:c:b:B:u:g:m:fhV";
extern char *__progname;
static const char *version_string =
"%s (Pygos %s) %s\n"
"Copyright (c) 2019 David Oberhollenzer\n"
"License ISC <https://opensource.org/licenses/ISC>\n"
"This is free software: you are free to change and redistribute it.\n"
"There is NO WARRANTY, to the extent permitted by law.\n"
"\n"
"Written by David Oberhollenzer.\n";
static const char *help_string =
"Usage: %s [OPTIONS] <package-file> <squashfs-file>\n"
"\n"
"Convert a package file into a squashfs image.\n"
"\n"
"Possible options:\n"
"\n"
" --compressor, -c <name> Select the compressor to use.\n"
" directories (defaults to 'xz').\n"
" --timestamp, -t <value> Set timestamp for all inodes and image file.\n"
" Defaults to 0 (Jan 1st, 1970).\n"
" --block-size, -b <size> Block size to use for Squashfs image.\n"
" Defaults to %u.\n"
" --dev-block-size, -B <size> Device block size to padd the image to.\n"
" Defaults to %u.\n"
" --default-uid, -u <valie> Default user ID for implicitly created\n"
" directories (defaults to 0).\n"
" --default-gid, -g <value> Default group ID for implicitly created\n"
" directories (defaults to 0).\n"
" --default-mode, -m <value> Default permissions for implicitly created\n"
" directories (defaults to 0755).\n"
" --force, -f Overwrite the output file if it exists.\n"
" --help, -h Print help text and exit.\n"
" --version, -V Print version information and exit.\n"
"\n";
static const char *compressors[] = {
[SQFS_COMP_GZIP] = "gzip",
[SQFS_COMP_LZMA] = "lzma",
[SQFS_COMP_LZO] = "lzo",
[SQFS_COMP_XZ] = "xz",
[SQFS_COMP_LZ4] = "lz4",
[SQFS_COMP_ZSTD] = "zstd",
};
static long read_number(const char *name, const char *str, long min, long max)
{
long base = 10, result = 0;
int x;
if (str[0] == '0') {
if (str[1] == 'x' || str[1] == 'X') {
base = 16;
str += 2;
} else {
base = 8;
}
}
if (!isxdigit(*str))
goto fail_num;
while (isxdigit(*str)) {
x = *(str++);
if (isupper(x)) {
x = x - 'A' + 10;
} else if (islower(x)) {
x = x - 'a' + 10;
} else {
x -= '0';
}
if (x >= base)
goto fail_num;
if (result > (LONG_MAX - x) / base)
goto fail_ov;
result = result * base + x;
}
if (result < min)
goto fail_uf;
if (result > max)
goto fail_ov;
return result;
fail_num:
fprintf(stderr, "%s: expected numeric value > 0\n", name);
goto fail;
fail_uf:
fprintf(stderr, "%s: number to small\n", name);
goto fail;
fail_ov:
fprintf(stderr, "%s: number to large\n", name);
goto fail;
fail:
exit(EXIT_FAILURE);
}
static int sqfs_padd_file(sqfs_info_t *info)
{
size_t padd_sz = info->super.bytes_used % info->dev_blk_size;
uint8_t *buffer;
ssize_t ret;
off_t off;
if (padd_sz == 0)
return 0;
off = lseek(info->outfd, 0, SEEK_END);
if (off == (off_t)-1) {
perror("seek on output file");
return -1;
}
padd_sz = info->dev_blk_size - padd_sz;
buffer = alloca(padd_sz);
memset(buffer, 0, padd_sz);
ret = write_retry(info->outfd, buffer, padd_sz);
if (ret < 0) {
perror("Error padding squashfs image to page size");
return -1;
}
if (ret == 0) {
fputs("Truncated write trying to padd squashfs image\n",
stderr);
return -1;
}
return 0;
}
int main(int argc, char **argv)
{
uint32_t blocksize = SQFS_DEFAULT_BLOCK_SIZE, timestamp = 0;
int i, outmode = O_WRONLY | O_CREAT | O_EXCL;
E_SQFS_COMPRESSOR compressor = SQFS_COMP_XZ;
const char *infile, *outfile;
int status = EXIT_FAILURE;
compressor_stream_t *cmp;
sqfs_info_t info;
memset(&info, 0, sizeof(info));
info.dev_blk_size = SQFS_DEVBLK_SIZE;
info.fs.default_uid = 0;
info.fs.default_gid = 0;
info.fs.default_mode = 0755;
for (;;) {
i = getopt_long(argc, argv, short_opts, long_opts, NULL);
if (i == -1)
break;
switch (i) {
case 't':
timestamp = read_number("Timestamp", optarg,
0, 0xFFFFFFFF);
break;
case 'c':
for (i = SQFS_COMP_MIN; i <= SQFS_COMP_MAX; ++i) {
if (strcmp(compressors[i], optarg) == 0) {
compressor = i;
break;
}
}
break;
case 'b':
blocksize = read_number("Block size", optarg,
1024, 0xFFFFFFFF);
break;
case 'B':
info.dev_blk_size = read_number("Device block size",
optarg, 4096,
0xFFFFFFFF);
break;
case 'u':
info.fs.default_uid = read_number("User ID", optarg,
0, 0xFFFFFFFF);
break;
case 'g':
info.fs.default_gid = read_number("Group ID", optarg,
0, 0xFFFFFFFF);
break;
case 'm':
info.fs.default_mode = read_number("Permissions",
optarg, 0, 0777);
break;
case 'f':
outmode = O_WRONLY | O_CREAT | O_TRUNC;
break;
case 'h':
printf(help_string, __progname,
SQFS_DEFAULT_BLOCK_SIZE, SQFS_DEVBLK_SIZE);
fputs("Available compressors:\n", stdout);
for (i = SQFS_COMP_MIN; i <= SQFS_COMP_MAX; ++i)
printf("\t%s\n", compressors[i]);
exit(EXIT_SUCCESS);
case 'V':
printf(version_string, __progname,
PACKAGE_NAME, PACKAGE_VERSION);
exit(EXIT_SUCCESS);
default:
goto fail_arg;
}
}
if ((optind + 1) >= argc) {
fputs("Missing arguments: input and output files.\n", stderr);
goto fail_arg;
}
infile = argv[optind++];
outfile = argv[optind++];
info.rd = pkg_reader_open(infile);
if (info.rd == NULL)
return EXIT_FAILURE;
info.outfd = open(outfile, outmode, 0644);
if (info.outfd < 0) {
perror(outfile);
goto out_pkg_close;
}
if (sqfs_super_init(&info.super, timestamp, blocksize, compressor))
goto out_close;
cmp = sqfs_get_compressor(&info.super);
if (cmp == NULL)
goto out_close;
info.block = malloc(info.super.block_size * 3);
if (info.block == NULL) {
perror("malloc");
goto out_cmp;
}
info.scratch = (char *)info.block + info.super.block_size;
info.fragment = (char *)info.block + 2 * info.super.block_size;
if (create_vfs_tree(&info))
goto out_buffer;
if (sqfs_super_write(&info.super, info.outfd))
goto out_tree;
if (pkg_data_to_sqfs(&info, cmp))
goto out_fragments;
free(info.block);
info.block = NULL;
if (sqfs_write_inodes(&info, cmp))
goto out_fragments;
if (sqfs_write_fragment_table(info.outfd, &info.super,
info.fragments, info.num_fragments,
cmp))
goto out_fragments;
if (sqfs_write_ids(info.outfd, &info.super,
info.fs.id_tbl, info.fs.num_ids, cmp))
goto out_fragments;
if (sqfs_super_write(&info.super, info.outfd))
goto out_fragments;
if (sqfs_padd_file(&info))
goto out_fragments;
status = EXIT_SUCCESS;
out_fragments:
free(info.fragments);
out_tree:
destroy_vfs_tree(&info.fs);
out_buffer:
free(info.block);
out_cmp:
cmp->destroy(cmp);
out_close:
close(info.outfd);
out_pkg_close:
pkg_reader_close(info.rd);
return status;
fail_arg:
fprintf(stderr, "Try `%s --help' for more information.\n", __progname);
return EXIT_FAILURE;
}

114
sqfs/pkg2sqfs.h
View file

@ -1,114 +0,0 @@
/* SPDX-License-Identifier: ISC */
#ifndef PKG2SQFS_H
#define PKG2SQFS_H
#include <sys/types.h>
#include <sys/stat.h>
#include <stdbool.h>
#include <unistd.h>
#include <stdlib.h>
#include <getopt.h>
#include <stdint.h>
#include <assert.h>
#include <limits.h>
#include <string.h>
#include <unistd.h>
#include <stdio.h>
#include <fcntl.h>
#include <errno.h>
#include <ctype.h>
#include "util/util.h"
#include "filelist/image_entry.h"
#include "comp/compressor.h"
#include "pkg/pkgreader.h"
#include "sqfs/squashfs.h"
#include "pkg/pkgio.h"
#include "config.h"
typedef struct file_info_t {
struct file_info_t *frag_next;
uint64_t size;
uint64_t startblock;
uint32_t id;
uint32_t fragment;
uint32_t fragment_offset;
uint32_t blocksizes[];
} file_info_t;
typedef struct {
struct node_t *children;
uint64_t dref;
size_t size;
} dir_info_t;
typedef struct node_t {
struct node_t *parent;
struct node_t *next;
const char *name;
union {
dir_info_t *dir;
const char *symlink;
dev_t device;
file_info_t *file;
} data;
uint64_t inode_ref;
uint32_t inode_num;
uint16_t type;
uint16_t mode;
uint16_t uid_idx;
uint16_t gid_idx;
uint8_t extra[];
} node_t;
typedef struct {
uint32_t default_uid;
uint32_t default_gid;
uint32_t default_mode;
node_t **node_tbl;
size_t num_inodes;
uint32_t *id_tbl;
size_t num_ids;
size_t max_ids;
node_t *root;
} vfs_t;
typedef struct {
int outfd;
sqfs_super_t super;
vfs_t fs;
pkg_reader_t *rd;
void *block;
void *scratch;
void *fragment;
sqfs_fragment_t *fragments;
size_t num_fragments;
size_t max_fragments;
int file_block_count;