Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

tgt_loop: Introduce device offset #89

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

tgt_loop: Introduce device offset #89

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
fd4d570
nbd_setup_tgt: Actually print @send_zc
zippy2 Jan 29, 2025
8539e8b
lib: Actually return ulong in ublksrv_json_read_target_ulong_info()
zippy2 Jan 29, 2025
2772eed
tgt_loop: Avoid using global variables
zippy2 Jan 30, 2025
0db142c
tgt_loop: Introduce device offset
zippy2 Jan 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion include/ublksrv.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -712,7 +712,7 @@ extern int ublksrv_json_read_target_str_info(const char *jbuf, int len,
* @param val field value with ulong type
*/
extern int ublksrv_json_read_target_ulong_info(const char *jbuf,
const char *name, long *val);
const char *name, unsigned long *val);

/**
* Serialize json buffer from target field with string type
Expand Down
2 changes: 1 addition & 1 deletion lib/ublksrv_json.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@ int ublksrv_json_read_target_str_info(const char *jbuf, int len,
}

int ublksrv_json_read_target_ulong_info(const char *jbuf,
const char *name, long *val)
const char *name, unsigned long *val)
{
json j;
std::string s;
Expand Down
4 changes: 2 additions & 2 deletions nbd/tgt_nbd.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -800,7 +800,7 @@ static int nbd_setup_tgt(struct ublksrv_dev *dev, int type, bool recovery,
char *tlshostname = NULL;
bool tls = false;

long send_zc = 0;
unsigned long send_zc = 0;


if (info->flags & UBLK_F_USER_COPY)
Expand All @@ -817,7 +817,7 @@ static int nbd_setup_tgt(struct ublksrv_dev *dev, int type, bool recovery,
exp_name);
ublksrv_json_read_target_ulong_info(jbuf, "send_zc", &send_zc);

NBD_HS_DBG("%s: host %s unix %s exp_name %s send_zc\n", __func__,
NBD_HS_DBG("%s: host %s unix %s exp_name %s send_zc: %lu\n", __func__,
host_name, unix_path, exp_name, send_zc);
for (i = 0; i < info->nr_hw_queues; i++) {
int sock;
Expand Down
85 changes: 66 additions & 19 deletions tgt_loop.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,11 @@
#include <sys/epoll.h>
#include "ublksrv_tgt.h"

static bool user_copy;
static bool block_device;
struct loop_tgt_data {
bool user_copy;
bool block_device;
unsigned long offset;
};

static bool backing_supports_discard(char *name)
{
Expand Down Expand Up @@ -41,9 +44,11 @@ static int loop_setup_tgt(struct ublksrv_dev *dev, int type, bool recovery,
const struct ublksrv_ctrl_dev_info *info =
ublksrv_ctrl_get_dev_info(ublksrv_get_ctrl_dev(dev));
int fd, ret;
long direct_io = 0;
unsigned long direct_io = 0;
struct ublk_params p;
char file[PATH_MAX];
struct loop_tgt_data *tgt_data = (struct loop_tgt_data*)dev->tgt.tgt_data;
struct stat sb;

ublk_assert(jbuf);

Expand All @@ -62,6 +67,14 @@ static int loop_setup_tgt(struct ublksrv_dev *dev, int type, bool recovery,
return ret;
}

ret = ublksrv_json_read_target_ulong_info(jbuf, "offset",
&tgt_data->offset);
if (ret) {
ublk_err( "%s: read target offset failed %d\n",
__func__, ret);
return ret;
}

ret = ublksrv_json_read_params(&p, jbuf);
if (ret) {
ublk_err( "%s: read ublk params failed %d\n",
Expand All @@ -76,6 +89,14 @@ static int loop_setup_tgt(struct ublksrv_dev *dev, int type, bool recovery,
return fd;
}

if (fstat(fd, &sb) < 0) {
ublk_err( "%s: unable to stat %s\n",
__func__, file);
return -1;
}

tgt_data->block_device = S_ISBLK(sb.st_mode);

if (direct_io)
fcntl(fd, F_SETFL, O_DIRECT);

Expand All @@ -84,10 +105,11 @@ static int loop_setup_tgt(struct ublksrv_dev *dev, int type, bool recovery,
tgt->tgt_ring_depth = info->queue_depth;
tgt->nr_fds = 1;
tgt->fds[1] = fd;
user_copy = info->flags & UBLK_F_USER_COPY;
if (user_copy)
tgt_data->user_copy = info->flags & UBLK_F_USER_COPY;
if (tgt_data->user_copy)
tgt->tgt_ring_depth *= 2;


return 0;
}

Expand All @@ -98,6 +120,8 @@ static int loop_recovery_tgt(struct ublksrv_dev *dev, int type)

ublk_assert(type == UBLKSRV_TGT_TYPE_LOOP);

dev->tgt.tgt_data = calloc(sizeof(struct loop_tgt_data), 1);

return loop_setup_tgt(dev, type, true, jbuf);
}

Expand All @@ -110,6 +134,7 @@ static int loop_init_tgt(struct ublksrv_dev *dev, int type, int argc, char
static const struct option lo_longopts[] = {
{ "file", 1, NULL, 'f' },
{ "buffered_io", no_argument, &buffered_io, 1},
{ "offset", required_argument, NULL, 'o'},
{ NULL }
};
unsigned long long bytes;
Expand Down Expand Up @@ -137,18 +162,22 @@ static int loop_init_tgt(struct ublksrv_dev *dev, int type, int argc, char
},
};
bool can_discard = false;
unsigned long offset = 0;

strcpy(tgt_json.name, "loop");

if (type != UBLKSRV_TGT_TYPE_LOOP)
return -1;

while ((opt = getopt_long(argc, argv, "-:f:",
while ((opt = getopt_long(argc, argv, "-:f:o:",
lo_longopts, NULL)) != -1) {
switch (opt) {
case 'f':
file = strdup(optarg);
break;
case 'o':
offset = strtoul(optarg, NULL, 10);
break;
}
}

Expand All @@ -174,12 +203,10 @@ static int loop_init_tgt(struct ublksrv_dev *dev, int type, int argc, char
return -1;
if (ioctl(fd, BLKPBSZGET, &pbs) != 0)
return -1;
block_device = true;
p.basic.logical_bs_shift = ilog2(bs);
p.basic.physical_bs_shift = ilog2(pbs);
can_discard = backing_supports_discard(file);
} else if (S_ISREG(st.st_mode)) {
block_device = false;
bytes = st.st_size;
can_discard = true;
p.basic.logical_bs_shift = ilog2(st.st_blksize);
Expand All @@ -199,6 +226,17 @@ static int loop_init_tgt(struct ublksrv_dev *dev, int type, int argc, char
buffered_io = 1;
}

if (bytes > 0) {
unsigned long long offset_bytes = offset * (1 << p.basic.physical_bs_shift);

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

From the helper message of offset skips first NUM sectors on backing file, here we should figure out offset_bytes via offset * 512, which is applied in IO code path too.

Otherwise, this PR looks good.

if (offset_bytes >= bytes) {
ublk_err( "%s: offset %lu greater than device size %llu",
__func__, offset, bytes);
return -2;
}
bytes -= offset_bytes;
}

tgt_json.dev_size = bytes;
p.basic.dev_sectors = bytes >> 9;

Expand All @@ -212,17 +250,21 @@ static int loop_init_tgt(struct ublksrv_dev *dev, int type, int argc, char
ublk_json_write_target_base(dev, &jbuf, &jbuf_size, &tgt_json);
ublk_json_write_tgt_str(dev, &jbuf, &jbuf_size, "backing_file", file);
ublk_json_write_tgt_long(dev, &jbuf, &jbuf_size, "direct_io", !buffered_io);
ublk_json_write_tgt_ulong(dev, &jbuf, &jbuf_size, "offset", offset);
ublk_json_write_params(dev, &jbuf, &jbuf_size, &p);

close(fd);

dev->tgt.tgt_data = calloc(sizeof(struct loop_tgt_data), 1);

return loop_setup_tgt(dev, type, false, jbuf);
}

static void loop_usage_for_add(void)
{
printf(" loop: -f backing_file [--buffered_io]\n");
printf(" loop: -f backing_file [--buffered_io] [--offset NUM]\n");
printf(" default is direct IO to backing file\n");
printf(" offset skips first NUM sectors on backing file\n");
}

static inline int loop_fallocate_mode(const struct ublksrv_io_desc *iod)
Expand Down Expand Up @@ -250,8 +292,9 @@ static void loop_queue_tgt_read(const struct ublksrv_queue *q,
const struct ublksrv_io_desc *iod, int tag)
{
unsigned ublk_op = ublksrv_get_op(iod);
const struct loop_tgt_data *tgt_data = (struct loop_tgt_data*) q->dev->tgt.tgt_data;

if (user_copy) {
if (tgt_data->user_copy) {
struct io_uring_sqe *sqe, *sqe2;
__u64 pos = ublk_pos(q->q_id, tag, 0);
void *buf = ublksrv_queue_get_io_buf(q, tag);
Expand All @@ -260,7 +303,7 @@ static void loop_queue_tgt_read(const struct ublksrv_queue *q,
io_uring_prep_read(sqe, 1 /*fds[1]*/,
buf,
iod->nr_sectors << 9,
iod->start_sector << 9);
(iod->start_sector + tgt_data->offset) << 9);
io_uring_sqe_set_flags(sqe, IOSQE_FIXED_FILE | IOSQE_IO_LINK);
sqe->user_data = build_user_data(tag, ublk_op, 1, 1);

Expand All @@ -277,7 +320,7 @@ static void loop_queue_tgt_read(const struct ublksrv_queue *q,
io_uring_prep_read(sqe, 1 /*fds[1]*/,
buf,
iod->nr_sectors << 9,
iod->start_sector << 9);
(iod->start_sector + tgt_data->offset) << 9);
io_uring_sqe_set_flags(sqe, IOSQE_FIXED_FILE);
sqe->user_data = build_user_data(tag, ublk_op, 0, 1);
}
Expand All @@ -287,8 +330,9 @@ static void loop_queue_tgt_write(const struct ublksrv_queue *q,
const struct ublksrv_io_desc *iod, int tag)
{
unsigned ublk_op = ublksrv_get_op(iod);
const struct loop_tgt_data *tgt_data = (struct loop_tgt_data*) q->dev->tgt.tgt_data;

if (user_copy) {
if (tgt_data->user_copy) {
struct io_uring_sqe *sqe, *sqe2;
__u64 pos = ublk_pos(q->q_id, tag, 0);
void *buf = ublksrv_queue_get_io_buf(q, tag);
Expand All @@ -301,7 +345,7 @@ static void loop_queue_tgt_write(const struct ublksrv_queue *q,

io_uring_prep_write(sqe2, 1 /*fds[1]*/,
buf, iod->nr_sectors << 9,
iod->start_sector << 9);
(iod->start_sector + tgt_data->offset) << 9);
io_uring_sqe_set_flags(sqe2, IOSQE_FIXED_FILE);
sqe2->rw_flags |= RWF_DSYNC;
/* bit63 marks us as tgt io */
Expand All @@ -314,7 +358,7 @@ static void loop_queue_tgt_write(const struct ublksrv_queue *q,
io_uring_prep_write(sqe, 1 /*fds[1]*/,
buf,
iod->nr_sectors << 9,
iod->start_sector << 9);
(iod->start_sector + tgt_data->offset) << 9);
io_uring_sqe_set_flags(sqe, IOSQE_FIXED_FILE);
sqe->rw_flags |= RWF_DSYNC;
/* bit63 marks us as tgt io */
Expand All @@ -328,13 +372,14 @@ static int loop_queue_tgt_io(const struct ublksrv_queue *q,
const struct ublksrv_io_desc *iod = data->iod;
struct io_uring_sqe *sqe;
unsigned ublk_op = ublksrv_get_op(iod);
const struct loop_tgt_data *tgt_data = (struct loop_tgt_data*) q->dev->tgt.tgt_data;

switch (ublk_op) {
case UBLK_IO_OP_FLUSH:
ublk_get_sqe_pair(q->ring_ptr, &sqe, NULL);
io_uring_prep_sync_file_range(sqe, 1 /*fds[1]*/,
iod->nr_sectors << 9,
iod->start_sector << 9,
(iod->start_sector + tgt_data->offset) << 9,
IORING_FSYNC_DATASYNC);
io_uring_sqe_set_flags(sqe, IOSQE_FIXED_FILE);
/* bit63 marks us as tgt io */
Expand All @@ -345,7 +390,7 @@ static int loop_queue_tgt_io(const struct ublksrv_queue *q,
ublk_get_sqe_pair(q->ring_ptr, &sqe, NULL);
io_uring_prep_fallocate(sqe, 1 /*fds[1]*/,
loop_fallocate_mode(iod),
iod->start_sector << 9,
(iod->start_sector + tgt_data->offset) << 9,
iod->nr_sectors << 9);
io_uring_sqe_set_flags(sqe, IOSQE_FIXED_FILE);
/* bit63 marks us as tgt io */
Expand Down Expand Up @@ -399,14 +444,15 @@ static int loop_handle_io_async(const struct ublksrv_queue *q,
const struct ublk_io_data *data)
{
struct ublk_io_tgt *io = __ublk_get_io_tgt_data(data);
const struct loop_tgt_data *tgt_data = (struct loop_tgt_data*) q->dev->tgt.tgt_data;

if (block_device && ublksrv_get_op(data->iod) == UBLK_IO_OP_DISCARD) {
if (tgt_data->block_device && ublksrv_get_op(data->iod) == UBLK_IO_OP_DISCARD) {
__u64 r[2];
int res;

io_uring_submit(q->ring_ptr);

r[0] = data->iod->start_sector << 9;
r[0] = (data->iod->start_sector + tgt_data->offset) << 9;
r[1] = data->iod->nr_sectors << 9;
res = ioctl(q->dev->tgt.fds[1], BLKDISCARD, &r);
ublksrv_complete_io(q, data->tag, res);
Expand Down Expand Up @@ -440,6 +486,7 @@ static void loop_deinit_tgt(const struct ublksrv_dev *dev)
{
fsync(dev->tgt.fds[1]);
close(dev->tgt.fds[1]);
free(dev->tgt.tgt_data);
}

struct ublksrv_tgt_type loop_tgt_type = {
Expand Down