利用 Windows 内建的 Driver 透過 IOCTL 發送 NVMe Command

Limitations

劈頭就要講限制因為 Windows 就是麻煩，不像在 Linux 上可以直接使用 linux-nvme/nvme-cli 就搞定。限制就是只有特定幾個 NVMe Admin Command 可以下，網路上的 VS Studio 專案可以直接拿來用，也有表格註明能用的 Command 們。但人生就是這個但是，我編譯不過啊@@ 只好自己寫再編譯，並且就有了本篇。

NVMe IO Command – NVM Command Set 的部分大都可以透過一對一的 SCSI Command [2] 來做到，因此以下主要講 Admin Command。

Sample Codes

我知道大家都只要這個。

For Generic Commands

#include <windows.h>
#include "nvme.h" //remember to include this

typedef VOID (*CQ_CALLBACK)(DWORD req_val, DWORD cdw0);
/*!
 * For Identify, Get Feature, Get Log Page Only
 * @param data_type can only be NVMeDataTypeIdentify, NVMeDataTypeFeature, or NVMeDataTypeLogPage
 * @param req_val request value means CNS, FID, or LID
 * @param req_sub_val
 * @param pdata used if additional data is request
 * @param xfer_len size of pdata in bytes
 * @param callback a void function takes 2 parameters: void (req_val, cdw0)
 */
DWORD nvme_specific(HANDLE FileHandle, STORAGE_PROTOCOL_NVME_DATA_TYPE data_type, DWORD req_val, DWORD req_sub_val, PVOID pdata, DWORD xfer_len, CQ_CALLBACK callback)
{
	BOOL result;
	PVOID buffer = NULL;
	ULONG bufferLength = 0;
	ULONG returnedLength = 0;

	PSTORAGE_PROPERTY_QUERY query = NULL;
	PSTORAGE_PROTOCOL_SPECIFIC_DATA protocolData = NULL;
	PSTORAGE_PROTOCOL_DATA_DESCRIPTOR protocolDataDescr = NULL;

	//
	// Allocate buffer for use.
	//
	bufferLength = FIELD_OFFSET(STORAGE_PROPERTY_QUERY, AdditionalParameters) + sizeof(STORAGE_PROTOCOL_SPECIFIC_DATA) + xfer_len;
	buffer = malloc(bufferLength);

	if (buffer == NULL)
	{
		printf("DeviceNVMeQueryProtocolDataTest: allocate buffer failed, exit.\n");
		return 1;
	}

	//
	// Initialize query data structure to get Identify Controller Data.
	//
	ZeroMemory(buffer, bufferLength);

	query = (PSTORAGE_PROPERTY_QUERY)buffer;
	protocolDataDescr = (PSTORAGE_PROTOCOL_DATA_DESCRIPTOR)buffer;
	protocolData = (PSTORAGE_PROTOCOL_SPECIFIC_DATA)query->AdditionalParameters;

	query->PropertyId = StorageAdapterProtocolSpecificProperty;
	query->QueryType = PropertyStandardQuery;

	protocolData->ProtocolType = ProtocolTypeNvme;
	protocolData->DataType = data_type;
	protocolData->ProtocolDataRequestValue = req_val;
	protocolData->ProtocolDataRequestSubValue = req_sub_val;
	if (xfer_len)
	{
		protocolData->ProtocolDataOffset = sizeof(STORAGE_PROTOCOL_SPECIFIC_DATA);
		protocolData->ProtocolDataLength = xfer_len;
	}

	//
	// Send request down.
	//
	result = DeviceIoControl(FileHandle,
							 IOCTL_STORAGE_QUERY_PROPERTY,
							 buffer,
							 bufferLength,
							 buffer,
							 bufferLength,
							 &returnedLength,
							 NULL);

	if (!result || (returnedLength == 0))
	{
		printf("FAIL, Error Code=%d\n", GetLastError());
		return GetLastError();
	}

	//
	// Validate the returned data.
	//
	if ((protocolDataDescr->Version != sizeof(STORAGE_PROTOCOL_DATA_DESCRIPTOR)) ||
		(protocolDataDescr->Size != sizeof(STORAGE_PROTOCOL_DATA_DESCRIPTOR)))
	{
		printf("Data descriptor header not valid\n");
		return 1;
	}

	protocolData = &protocolDataDescr->ProtocolSpecificData;
	memcpy_s(pdata, xfer_len, (PCHAR)protocolData + protocolData->ProtocolDataOffset, xfer_len);

	if (callback != NULL)
	{
		callback(req_val, protocolDataDescr->ProtocolSpecificData.FixedProtocolReturnData);
	}

	free(buffer);
	return 0;
}

然後就可以下個 Identify 之類：

DWORD nvme_identify()
{
	HANDLE FileHandle = CreateFileA(
		"\\\\.\\physicaldrive0", GENERIC_WRITE | GENERIC_READ,
        FILE_SHARE_READ | FILE_SHARE_WRITE,
        NULL, OPEN_EXISTING, 0, NULL
    );
	CHAR pdata[NVME_MAX_LOG_SIZE];
	if (nvme_specific(FileHandle, NVMeDataTypeIdentify, NVME_IDENTIFY_CNS_CONTROLLER, 0, pdata, NVME_MAX_LOG_SIZE, NULL)) {
		return 1;
	}

	PNVME_IDENTIFY_CONTROLLER_DATA identifyControllerData = (PNVME_IDENTIFY_CONTROLLER_DATA)pdata;

	printf("[IDENTIFY] vid         : 0x%02X", identifyControllerData->VID);
	printf("[IDENTIFY] nn          : 0x%02X", identifyControllerData->NN);

	unsigned char str[41];
	memcpy(str, identifyControllerData->SN, 20);
	str[20] = '\0';
	printf("[IDENTIFY] serial_num  : %s\r\n", str);
	memcpy(str, identifyControllerData->MN, 40);
	str[40] = '\0';
	printf("[IDENTIFY] model_num   : %s\r\n", str);
	memcpy(str, identifyControllerData->FR, 8);
	str[8] = '\0';
	printf("[IDENTIFY] firmware_rev: %s\r\n", str);
	return 0;
}

或者 Get Log Page：

DWORD nvme_get_log_page(NVME_LOG_PAGES lid)
{
	HANDLE FileHandle = CreateFileA(
		"\\\\.\\physicaldrive0", GENERIC_WRITE | GENERIC_READ,
        FILE_SHARE_READ | FILE_SHARE_WRITE,
        NULL, OPEN_EXISTING, 0, NULL
    );
    CHAR pdata[NVME_MAX_LOG_SIZE];
	if(nvme_specific(FileHandle, NVMeDataTypeLogPage, lid, 0, pdata, NVME_MAX_LOG_SIZE, NULL)) {
		return 1;
	}
    switch(lid)
    {
        case NVME_LOG_PAGE_HEALTH_INFO:
            PNVME_HEALTH_INFO_LOG smartInfo = (PNVME_HEALTH_INFO_LOG)pdata;
            printf("SMART/Health Info - Temperature %d.\n", ((ULONG)smartInfo->Temperature[1] << 8 | smartInfo->Temperature[0]) - 273);
    }
	return 0;
}

或者 Get Feature：

void nvme_fid_callback(DWORD fid, DWORD cdw0)
{
	printf("[GET FEATURE] ");
	switch (fid)
	{
	case NVME_FEATURE_POWER_MANAGEMENT:
		printf("PS=%d\r\n", cdw0);
		break;
	default:
		printf("CDW0=%d\r\n", cdw0);
		break;
	}
};

DWORD nvme_get_feature(NVME_FEATURES fid)
{
	HANDLE FileHandle = CreateFileA(
		"\\\\.\\physicaldrive0", GENERIC_WRITE | GENERIC_READ,
        FILE_SHARE_READ | FILE_SHARE_WRITE,
        NULL, OPEN_EXISTING, 0, NULL
    );
	return nvme_specific(FileHandle, NVMeDataTypeFeature, fid, 0, NULL, 0, nvme_fid_callback);
}

For VUC Commands

VUC 的話（OPC =0xC0~0xFF）可以透過以下函式呼叫。

/*!
 * For VUC command only, OPC ranges 0xC0-0xFF
 * @param data_type can only be NVMeDataTypeIdentify, NVMeDataTypeFeature, or NVMeDataTypeLogPage
 * @param sqe the standard 64-byte NVMe Submission Queue Entry
 * @param prtc data transfer direction, can be NVME_PROTOCOL_NON_DATA, NVME_PROTOCOL_DATA_IN, or NVME_PROTOCOL_DATA_OUT
 * @param pdata used if additional data is request
 * @param xfer_len size of pdata in bytes
 */
DWORD nvme_vuc(HANDLE FileHandle, PNVME_COMMAND sqe, NVME_PROTOCOLS prtc, PCHAR pdata, DWORD xfer_len)
{
	BOOL result;
	PVOID buffer = NULL;
	ULONG bufferLength = 0;
	ULONG returnedLength = 0;

	PSTORAGE_PROTOCOL_COMMAND protocolCommand = NULL;
	PNVME_COMMAND command = NULL;

	//
	// Allocate buffer for use.
	//
	bufferLength = sizeof(STORAGE_PROTOCOL_COMMAND) + STORAGE_PROTOCOL_COMMAND_LENGTH_NVME + sizeof(NVME_ERROR_INFO_LOG) + xfer_len;
	buffer = malloc(bufferLength);

	if (buffer == NULL)
	{
		printf("DeviceNVMeQueryProtocolDataTest: allocate buffer failed, exit.\n");
		return 0;
	}

	ZeroMemory(buffer, bufferLength);
	protocolCommand = (PSTORAGE_PROTOCOL_COMMAND)buffer;

	protocolCommand->Version = STORAGE_PROTOCOL_STRUCTURE_VERSION;
	protocolCommand->Length = sizeof(STORAGE_PROTOCOL_COMMAND);
	protocolCommand->ProtocolType = ProtocolTypeNvme;
	protocolCommand->Flags = STORAGE_PROTOCOL_COMMAND_FLAG_ADAPTER_REQUEST;
	protocolCommand->CommandLength = STORAGE_PROTOCOL_COMMAND_LENGTH_NVME;
	protocolCommand->ErrorInfoLength = sizeof(NVME_ERROR_INFO_LOG);
	protocolCommand->TimeOutValue = 10;
	protocolCommand->ErrorInfoOffset = FIELD_OFFSET(STORAGE_PROTOCOL_COMMAND, Command) + STORAGE_PROTOCOL_COMMAND_LENGTH_NVME;
	protocolCommand->CommandSpecific = STORAGE_PROTOCOL_SPECIFIC_NVME_ADMIN_COMMAND;
	if (prtc == NVME_PROTOCOL_DATA_IN)
	{
		protocolCommand->DataFromDeviceTransferLength = xfer_len;
		protocolCommand->DataFromDeviceBufferOffset = protocolCommand->ErrorInfoOffset + protocolCommand->ErrorInfoLength;
	}
	else if (prtc == NVME_PROTOCOL_DATA_OUT)
	{
		protocolCommand->DataToDeviceTransferLength = xfer_len;
		protocolCommand->DataToDeviceBufferOffset = protocolCommand->ErrorInfoOffset + protocolCommand->ErrorInfoLength;
    memcpy_s((PCHAR)buffer + protocolCommand->DataToDeviceBufferOffset, xfer_len, pdata, xfer_len);
	}
	memcpy_s(protocolCommand->Command, STORAGE_PROTOCOL_COMMAND_LENGTH_NVME, sqe, STORAGE_PROTOCOL_COMMAND_LENGTH_NVME);

	//
	// Send request down.
	//
	result = DeviceIoControl(FileHandle,
							 IOCTL_STORAGE_PROTOCOL_COMMAND,
							 buffer,
							 bufferLength,
							 buffer,
							 bufferLength,
							 &returnedLength,
							 NULL);

	if (protocolCommand->ReturnStatus != STORAGE_PROTOCOL_STATUS_SUCCESS)
	{
		PNVME_ERROR_INFO_LOG err = (PNVME_ERROR_INFO_LOG)((PCHAR)buffer + protocolCommand->ErrorInfoOffset);
		printf("Fail, Return Status=%d, ", protocolCommand->ReturnStatus);
		printf("SCT=0x%02X, SC=0x%02X\n", err->Status.SCT, err->Status.SC);
		return err->Status.AsUshort;
	}
	else
	{
		printf("PASS\r\n");
		if (prtc == NVME_PROTOCOL_DATA_IN)
		{
			memcpy_s(pdata, xfer_len, (PCHAR)buffer + protocolCommand->DataFromDeviceBufferOffset, xfer_len);
		}
	}

	return 0;
}

前提是 Controller 要支援 Get Log Page – Commands Supported and Effects 頁且相應的 VUC Opcode 有描述正確。

Opcode 的 Bit0, Bit1 要符合 Spec 規範，否則 Data 傳輸會有問題

然後就可以:

NVME_COMMAND sq = {0};
sq.CDW0.OPC = opc;
sq.u.GENERAL.CDW10 = 0;
sq.u.GENERAL.CDW11 = 0;
sq.u.GENERAL.CDW12 = 0;
sq.u.GENERAL.CDW13 = 0;
sq.u.GENERAL.CDW14 = 0;
sq.u.GENERAL.CDW15 = 0;
nvme_vuc(FileHandle, &sq, (NVME_PROTOCOLS)protocol, NULL, 0);

雖然看起來有點囉嗦（真的是滿囉嗦的），但這樣我們就可以下 NVMe Admin Command 惹，也只有那幾個基本的呵呵。

編譯就懶得用 Makefile 了，重點是還要另外裝啊啊啊，Windows 哎~

g++ main.cpp -o test.exe

其他 NVMe Command

你如果在好奇，想下前面提到的 Identify、Get Log Page、Get Feature、VUC 以外的 Command 怎麼辦？微軟提供了另一種迂迴方式讓你走，SCSI Translation，也就是要你對 NVMe 磁碟機下 SCSI 指令，內建的 Driver 再根據 Spec 幫你轉成對應的 NVMe Command。

提供大家文件參考，就不再贅述具體的實作細節。

Reference

  1. Working with NVMe drives | Microsoft Docs
  2. NVM Express: SCSI Translation Reference

 

Post Views: 4,047