1
     ˆÄÃ„e¶šgÂÁÀ ?÷     /* SPDX-License-Identifier: GPL-2.0 */
#ifndef BLK_MQ_H
#define BLK_MQ_H

#include <linux/blkdev.h>
#include <linux/sbitmap.h>
#include <linux/lockdep.h>
#include <linux/scatterlist.h>
#include <linux/prefetch.h>
#include <linux/srcu.h>
#include <linux/rw_hint.h>
#include <linux/rwsem.h>

struct blk_mq_tags;
struct blk_flush_queue;

#define BLKDEV_MIN_RQ	4
#define BLKDEV_DEFAULT_RQ	128

enum rq_end_io_ret {
	RQ_END_IO_NONE,
	RQ_END_IO_FREE,
};

typedef enum rq_end_io_ret (rq_end_io_fn)(struct request *, blk_status_t);

/*
 * request flags */
typedef __u32 __bitwise req_flags_t;

/* Keep rqf_name[] in sync with the definitions below */
enum rqf_flags {
	/* drive already may have started this one */
	__RQF_STARTED,
	/* request for flush sequence */
	__RQF_FLUSH_SEQ,
	/* merge of different types, fail separately */
	__RQF_MIXED_MERGE,
	/* don't call prep for this one */
	__RQF_DONTPREP,
	/* use hctx->sched_tags */
	__RQF_SCHED_TAGS,
	/* use an I/O scheduler for this request */
	__RQF_USE_SCHED,
	/* vaguely specified driver internal error.  Ignored by block layer */
	__RQF_FAILED,
	/* don't warn about errors */
	__RQF_QUIET,
	/* account into disk and partition IO statistics */
	__RQF_IO_STAT,
	/* runtime pm request */
	__RQF_PM,
	/* on IO scheduler merge hash */
	__RQF_HASHED,
	/* track IO completion time */
	__RQF_STATS,
	/* Look at ->special_vec for the actual data payload instead of the
	   bio chain. */
	__RQF_SPECIAL_PAYLOAD,
	/* request completion needs to be signaled to zone write plugging. */
	__RQF_ZONE_WRITE_PLUGGING,
	/* ->timeout has been called, don't expire again */
	__RQF_TIMED_OUT,
	__RQF_RESV,
	__RQF_BITS
};

#define RQF_STARTED		((__force req_flags_t)(1 << __RQF_STARTED))
#define RQF_FLUSH_SEQ		((__force req_flags_t)(1 << __RQF_FLUSH_SEQ))
#define RQF_MIXED_MERGE		((__force req_flags_t)(1 << __RQF_MIXED_MERGE))
#define RQF_DONTPREP		((__force req_flags_t)(1 << __RQF_DONTPREP))
#define RQF_SCHED_TAGS		((__force req_flags_t)(1 << __RQF_SCHED_TAGS))
#define RQF_USE_SCHED		((__force req_flags_t)(1 << __RQF_USE_SCHED))
#define RQF_FAILED		((__force req_flags_t)(1 << __RQF_FAILED))
#define RQF_QUIET		((__force req_flags_t)(1 << __RQF_QUIET))
#define RQF_IO_STAT		((__force req_flags_t)(1 << __RQF_IO_STAT))
#define RQF_PM			((__force req_flags_t)(1 << __RQF_PM))
#define RQF_HASHED		((__force req_flags_t)(1 << __RQF_HASHED))
#define RQF_STATS		((__force req_flags_t)(1 << __RQF_STATS))
#define RQF_SPECIAL_PAYLOAD	\
			((__force req_flags_t)(1 << __RQF_SPECIAL_PAYLOAD))
#define RQF_ZONE_WRITE_PLUGGING	\
			((__force req_flags_t)(1 << __RQF_ZONE_WRITE_PLUGGING))
#define RQF_TIMED_OUT		((__force req_flags_t)(1 << __RQF_TIMED_OUT))
#define RQF_RESV		((__force req_flags_t)(1 << __RQF_RESV))

/* flags that prevent us from merging requests: */
#define RQF_NOMERGE_FLAGS \
	(RQF_STARTED | RQF_FLUSH_SEQ | RQF_SPECIAL_PAYLOAD)

enum mq_rq_state {
	MQ_RQ_IDLE		= 0,
	MQ_RQ_IN_FLIGHT		= 1,
	MQ_RQ_COMPLETE		= 2,
};

/*
 * Try to put the fields that are referenced together in the same cacheline.
 *
 * If you modify this structure, make sure to update blk_rq_init() and
 * especially blk_mq_rq_ctx_init() to take care of the added fields.
 */
struct request {
	struct request_queue *q;
	struct blk_mq_ctx *mq_ctx;
	struct blk_mq_hw_ctx *mq_hctx;

	blk_opf_t cmd_flags;		/* op and common flags */
	req_flags_t rq_flags;

	int tag;
	int internal_tag;

	unsigned int timeout;

	/* the following two fields are internal, NEVER access directly */
	unsigned int __data_len;	/* total data len */
	sector_t __sector;		/* sector cursor */

	struct bio *bio;
	struct bio *biotail;

	union {
		struct list_head queuelist;
		struct request *rq_next;
	};

	struct block_device *part;
#ifdef CONFIG_BLK_RQ_ALLOC_TIME
	/* Time that the first bio started allocating this request. */
	u64 alloc_time_ns;
#endif
	/* Time that this request was allocated for this IO. */
	u64 start_time_ns;
	/* Time that I/O was submitted to the device. */
	u64 io_start_time_ns;

#ifdef CONFIG_BLK_WBT
	unsigned short wbt_flags;
#endif
	/*
	 * rq sectors