今天繼續(xù)liblightnvm，咱今天牛逼了，搞一下OCSSD的IO操作，具體也是一樣浓体，從tests里面的test_addr_io.c下手.
幾點(diǎn)注意：
- nvm_buf_alloc：讀寫的數(shù)據(jù)長(zhǎng)度需要經(jīng)過(guò)對(duì)齊霹疫。
最終調(diào)用的是posix_memalign.
void *nvm_buf_alloc(const struct nvm_geo *geo, size_t nbytes)
{
char *buf;
int ret;
if (!nbytes) {
errno = EINVAL;
return NULL;
}
ret = posix_memalign((void **)&buf, geo->sector_nbytes, nbytes);
//拿到了一個(gè)'至少'nbytes字節(jié)的內(nèi)存塊汁胆，并且這塊內(nèi)存起始地址是geo->sector_nbytes的倍數(shù).
//至于為什么需要以geo->sector_nbytes大小對(duì)齊勤婚，我推測(cè)原因可能有2：
//1.性能：讀寫時(shí)避免單次內(nèi)存操作被拆成2次.
//2.沒有作對(duì)齊的內(nèi)存塊在Lightnvm內(nèi)核進(jìn)行I/O時(shí)可能會(huì)出錯(cuò).
if (ret) {
errno = ret;
return NULL;
}
return buf;
}
寫入/讀取的模式 : with or without meta ?
寫入/讀取的模式 : plane access mode : Single-plane, Dual-plane or Quad-plane ?
由于是NAND-Based Flash Mem，Write前需要先Erase. 而Erase的單位一般是Block.
一些小測(cè)試（To be continue...）
a. Erase:
(1). 目前貌似Erase只能同時(shí)對(duì)一個(gè)lun上的所有plane進(jìn)行擦除.(Test1)
(2). 正如Paper上說(shuō)的勃痴，write任意一個(gè)地址之前，如果這個(gè)地址被寫過(guò)磷蜀，需要先擦除再寫入.(Test2)
b. Write:
(0). write/read的最小單元是sector.(實(shí)際上也是nvm_addr所能尋址的最小單元)
c. Write/Read with meta. 這里的meta是啥東西召耘？
一個(gè)猜想：OCSSD里面一個(gè)sector(Update: 目前只能確定page，sector還不能確定)存儲(chǔ)單元（storage unit）配套存在一個(gè)meta存儲(chǔ)單元. 帶外數(shù)據(jù)(out-of-bound data)可能指的就是這樣的每個(gè)尋址單位其對(duì)應(yīng)的一小塊字節(jié)區(qū).
d. Access mode: Sngl/Dual/Quad
一個(gè)猜想：由于之前推斷出的結(jié)構(gòu)褐隆，加上這個(gè)測(cè)試示范Dual和Quad 2個(gè)模式都是在nplane大于相應(yīng)的個(gè)數(shù)時(shí)才做的（Dual：2污它， Quad：4），很可能這里的access mode是單次I/O同時(shí)發(fā)向不同的plane庶弃，可以使I/O并發(fā).
附：
#include <liblightnvm.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
static char nvm_dev_path[NVM_DEV_PATH_LEN] = "/dev/nvme0n1";
static struct nvm_dev *dev;
static const struct nvm_geo *geo;
static struct nvm_addr a0;
static int channel = 0;
static int lun = 0;
static int plane = 0;
static int block = 10;
#define STRLEN 20
static char Str[STRLEN] = "OCSSD Test";
static void *Buf_for_read = NULL;
#define FAIL_ERASE do{ printf("Erase Failed\n"); }while(0)
#define FAIL_ALLOC do{ printf("Alloc Failed\n"); }while(0)
#define FAIL_WRITE do{ printf("Write Failed\n"); }while(0)
#define FAIL_READ do{ printf("Read Failed\n"); }while(0)
#define THE_SAME do{ printf("Same\n"); }while(0)
#define NOT_THE_SAME do{ printf("Not Same\n"); }while(0)
int setup(void)
{
dev = nvm_dev_open(nvm_dev_path);
if (!dev) {
perror("nvm_dev_open");
}
geo = nvm_dev_get_geo(dev);
a0.ppa = 0;
a0.g.ch = channel;
a0.g.lun = lun;
a0.g.pl = plane;
a0.g.blk = block;
Buf_for_read = nvm_buf_alloc(geo, geo->sector_nbytes);
if (!Buf_for_read) {
FAIL_ALLOC;
return -1;
}
return 0;
}
int teardown(void)
{
nvm_dev_close(dev);
if ( Buf_for_read ) {
free(Buf_for_read);
}
return 0;
}
uint64_t alignblk(struct nvm_addr adr)
{
struct nvm_addr alg;
alg.ppa = adr.ppa;
alg.g.pg = 0;
alg.g.sec = 0;
return alg.ppa;
}
void EraseNpl_1Blk(struct nvm_addr wh)//
{
struct nvm_ret ret;
ssize_t res;
int pmode = NVM_FLAG_PMODE_SNGL;
const int npl = geo->nplanes;
struct nvm_addr whichblk[npl];
for(int i = 0; i < npl; ++i){
whichblk[i].ppa = alignblk(wh);
whichblk[i].g.pl = i;
}
res = nvm_addr_erase(dev, whichblk, npl, pmode, &ret);//Erase 1 block of all planes inside a lun.
if(res < 0){
FAIL_ERASE;
nvm_ret_pr(&ret);
}
}
//pmode = Single-plane, without meta
void Write_1Sector(struct nvm_addr wh)
{
struct nvm_ret ret;
ssize_t res;
int pmode = NVM_FLAG_PMODE_SNGL;
void *bufptr = NULL;
bufptr = nvm_buf_alloc(geo, geo->sector_nbytes);//sector size
if(!bufptr){
FAIL_ALLOC;
goto OUT;
}
memcpy(bufptr, Str, STRLEN);
//2. write
res = nvm_addr_write(dev, &wh, 1, bufptr, NULL, pmode, &ret);//Write 1 sector
if(res < 0){
FAIL_WRITE;
}
free(bufptr);
OUT:
if(res < 0){
nvm_ret_pr(&ret);
}
return;
}
void Read_1Sector(struct nvm_addr wh)
{
struct nvm_ret ret;
ssize_t res;
int pmode = NVM_FLAG_PMODE_SNGL;
res = nvm_addr_read(dev, &wh, 1, Buf_for_read, NULL, pmode, &ret);
if(res < 0){
FAIL_READ;
nvm_ret_pr(&ret);
}
}
int MemCmp(unsigned char *a, unsigned char *b, int len)
{
for(int i = 0; i < len; ++i){
if(a[i] != b[i]){
return -1;
}
}
return 0;
}
void test_basic(void);
void test_write_no_erase(void)
{
struct nvm_addr addr;
addr.ppa = a0.ppa;
addr.g.pg = 1;
addr.g.sec = 1; //same as test_basic()'s address
printf("Before test: ");
test_basic();
printf("Run test: ");
Write_1Sector(addr);
}
void test_erase_1pl_1blk(void)
{
struct nvm_addr addr;
struct nvm_ret ret;
ssize_t res;
int pmode = NVM_FLAG_PMODE_SNGL;
const int npl = geo->nplanes;
struct nvm_addr whichblk[npl];
addr.ppa = a0.ppa;
for(int i = 0; i < npl; ++i){
whichblk[i].ppa = alignblk(addr);
whichblk[i].g.pl = i;
}
res = nvm_addr_erase(dev, whichblk, 1, pmode, &ret);//Erase 1 block of 1 planes inside a lun.
if(res < 0){
FAIL_ERASE;
nvm_ret_pr(&ret);
}
}
void test_basic(void)
{
struct nvm_addr addr;
addr.ppa = a0.ppa;
addr.g.pg = 1;
addr.g.sec = 1;
EraseNpl_1Blk(addr);
Write_1Sector(addr);
Read_1Sector(addr);
if(0 == MemCmp(Str, Buf_for_read, strlen(Str))){
THE_SAME;
}else{
NOT_THE_SAME;
}
}
typedef void (* FuncType) (void);
void RunTests()
{
FuncType tests[] = {
test_basic,
test_erase_1pl_1blk,
test_write_no_erase
};
const char *teststr[] = {
"test_basic",
"test_erase_1pl_1blk",
"test_write_no_erase"
};
for(int i = 0; i < 3; i++){
printf("====Test %d====\n %s:\n", i, teststr[i]);
tests[i]();
}
}
int main()
{
if( setup() < 0){
return -1;
}
RunTests();
teardown();
return 0;
}
Result:
====Test 0====
test_basic:
Same
====Test 1====
test_erase_1pl_1blk:
Erase Failed
nvm_ret { result(0x2), status(0) }
====Test 2====
test_write_no_erase:
Before test: Same
Run test: Write Failed
nvm_ret { result(0x2), status(0) }
Qemu Output:
lnvm: Erase not performed to all planes (1)
Erased failed
ppa:ch:0,lun:0,blk:10,pg:0,pl:0,sec:0
Attempting to write to non erased block (172)
lnvm: set written status failed
ppa:ch:0,lun:0,blk:10,pg:1,pl:1,sec:0
PS
這個(gè)lib讓使用qemu的用戶可以直接通過(guò)nvme與OCSSD交互衫贬，不需要通過(guò)內(nèi)核lightnvm模塊的target（包括需要初始化（lnvm init）/生成一個(gè)target設(shè)備（lnvm create）等）
PPS
我在測(cè)試write with meta時(shí)發(fā)現(xiàn)meta存在寫入與讀取不匹配的情況，后來(lái)看了一眼最新的qemu-nvme歇攻，發(fā)現(xiàn)這個(gè)bug已經(jīng)在2天前的commit（3839a1f059fff）fix掉了. ～～～lucky! 改天可以分析一下這整個(gè)從用戶態(tài)lib到kerne再到hardware（qemu）的流程固惯。
lightnvm: fix bad dma write on metadata
[master](https://github.com/OpenChannelSSD/qemu-nvme)
1 parent commit 3839a1f059fff1dd40d8696b8d12bdebe256112a
Javier González committed 2 days ago
