|
前些日子, 测试提单, 直接写文件的情形下, 日志生成太慢, 影响体验 老代码(历史遗留代码): 尝试过各种可能问题的修改, 都不见效, 最后考虑换函数, 改用C++的ofstream: …… |
|
| 1分 |
这不科学啊。
|
| 2分 |
有缓冲的IO函数会好一些。
|
| 2分 |
摒弃fstream
使用FILE * |
|
换固态硬盘再试试？
|
|
|
现实就是这样啊 |
|
|
下午再重新写个测试 给各位看看
|
|
| 1分 |
难道fstream……….有更优秀的实现？
|
| 2分 |
因为ofstream的flush清的是自己的缓存，东西可能留在os的缓存里没写进磁盘。当然会更快……
|
|
非常感谢你的解答! |
|
|
下午又写找时间偷偷写了个测试代码:
#ifdef _MSC_VER
#include <sys/stat.h>
#include <io.h>
#include <Windows.h>
#else
#include <unistd.h>
#endif
#include <fcntl.h>
#include <ctime>
#include <cstdio>
#include <cstdlib>
#include <fstream>
#ifdef _MSC_VER
#define OPEN(fd, file) \
fd = open(file, O_CREAT | O_RDWR | O_APPEND, S_IWRITE | S_IREAD)
#define WRITE(fd, data, len) \
_write(fd, pszData, iDataLen)
#define FLUSH(fd) \
_commit(fd)
#define CLOSE(fd) \
_close(fd)
#define SLEEP(s) \
Sleep(1000 * s)
#else
#define OPEN(fd, file) \
umask(002); \
fd = open(file, O_CREAT | O_RDWR | O_APPEND, S_IRUSR | S_IWUSR | S_IRGRP)
#define WRITE(fd, data, len) \
write(fd, pszData, iDataLen);
#define FLUSH(fd) \
fsync(fd)
#define CLOSE(fd) \
close(fd)
#define SLEEP(s) \
sleep(s)
#endif
static int s_iFd = -1;
static FILE * s_pFile = NULL;
static std::ofstream s_ofs;
enum TEST_TYPE
{
TYPE_MIN,
TYPE_HANDLE = TYPE_MIN,
TYPE_FILE,
TYPE_OFSTREAM,
TYPE_MAX
};
const char * GetType(enum TEST_TYPE enType)
{
switch (enType)
{
case TYPE_HANDLE:
{
return("handle");
}
case TYPE_FILE:
{
return("FILE");
}
case TYPE_OFSTREAM:
{
return("ofstream");
}
default:
{
return("unknown");
}
}
};
void OpenFile(enum TEST_TYPE enType)
{
switch (enType)
{
case TYPE_HANDLE:
{
OPEN(s_iFd, "a.txt");
if (s_iFd < 0)
{
fprintf(stderr, "open failed\n");
exit(1);
}
break;
}
case TYPE_FILE:
{
s_pFile = fopen("b.txt", "w+");
if (NULL == s_pFile)
{
fprintf(stderr, "fopen failed\n");
exit(2);
}
break;
}
case TYPE_OFSTREAM:
{
s_ofs.open("c.txt", std::ios::binary | std::ios::app);
if (!s_ofs)
{
fprintf(stderr, "ofstream::open failed\n");
exit(3);
}
break;
}
default:
{
fprintf(stderr, "open: unknown type\n");
exit(4);
break;
}
}
}
void WriteFile(enum TEST_TYPE enType, const char * pszData, int iDataLen)
{
if (NULL == pszData || 0 >= iDataLen)
{
return;
}
switch (enType)
{
case TYPE_HANDLE:
{
while (0 < iDataLen)
{
int iWirteLen = WRITE(s_iFd, pszData, iDataLen);
if (0 > iWirteLen)
{
fprintf(stderr, "write failed\n");
break;
}
pszData += iWirteLen;
iDataLen -= iWirteLen;
}
FLUSH(s_iFd);
break;
}
case TYPE_FILE:
{
while (0 < iDataLen)
{
int iWirteLen = fwrite(pszData, iDataLen, 1, s_pFile);
if (0 > iWirteLen)
{
fprintf(stderr, "fwrite failed\n");
break;
}
pszData += iWirteLen;
iDataLen -= iWirteLen;
}
fflush(s_pFile);
break;
}
case TYPE_OFSTREAM:
{
s_ofs.write(pszData, iDataLen);
if (s_ofs.fail())
{
fprintf(stderr, "ofstream::write failed\n");
s_ofs.clear();
}
s_ofs.flush();
break;
}
default:
{
fprintf(stderr, "write: unknown type\n");
exit(5);
break;
}
}
}
void CloseFile(enum TEST_TYPE enType)
{
switch (enType)
{
case TYPE_HANDLE:
{
CLOSE(s_iFd);
break;
}
case TYPE_FILE:
{
fclose(s_pFile);
break;
}
case TYPE_OFSTREAM:
{
s_ofs.close();
break;
}
default:
{
fprintf(stderr, "close: unknown type\n");
exit(6);
break;
}
}
}
int main(int argc, char * argv[])
{
const char szData[] = "1234567890123456789012345678901234567890123456789012345678901234567890";
const int iDataLen = sizeof(szData) - 1;
enum TEST_TYPE enIndex = TYPE_MIN;
while (TYPE_MAX > enIndex)
{
const char * pszType = GetType(enIndex);
OpenFile(enIndex);
int iWriteCount = 0;
int iPrintCount = 0;
time_t tStartTime = time(NULL);
time_t tLastTime = tStartTime;
while (10 > iPrintCount)
{
WriteFile(enIndex, szData, iDataLen);
++iWriteCount;
if (time(NULL) > tLastTime)
{
++iPrintCount;
tLastTime = time(NULL);
fprintf(stdout, "type: %9s, speed: %9d\n", pszType,
(iWriteCount / (tLastTime - tStartTime)));
}
}
CloseFile(enIndex);
enIndex = static_cast<enum TEST_TYPE>(enIndex + 1);
SLEEP(2);
};
return(0);
}
|
|
|
下面是各平台的测试结果:
Linux: no-flush: flush: Windows: no-flush: flush: 依据这些打印数据, 可知: 有一点问题: 当然, 也许是我用错了函数导致的这种结果, 如果你有更合理的函数, 请告知! 非常感谢! PS: 现在CSDN贴的代码真丑! |
|
|
如果是突然断电，os缓存没有flush的话，你_write+_commit的会保存到磁盘，ofstream的就不会。而只是程序异常退出的话，两种都会保存到磁盘。
就看你要怎样程度的健壮性了。不存在ofstream秒杀另两个的说法，因为ofstream没有另两种健壮。 |
|
|
看完10楼的回复 才明白过来你说的是什么意思 |
|
|
谢谢你的讲解, 另外问一下: |
|
| 1分 |
缓存对性能有质的飞跃, 前提是你的程序最好别挂掉。
|
|
Handle的方式是不行了, 以后如果发现ofstream不行的话, 就改用FILE方式的, 谢谢 |
|
|
说明还有数据在缓存里啊。要么fstream的缓存要么os的缓存。 |
|
| 1分 |
信赵老师全身打滚。c库的FILE相关的操作也是有（和os缓存不同的）缓存的，用fflush就和你的ofstream.flush一样，不保证写入磁盘。 |
|
是的, 不过程序意外挂掉, 缓不缓冲, 日志都会有丢失的… |
|
|
但是 WINDOWS下: |
|
|
是否说明, WINDOWS下, FILE方式比ofstream可靠?
|
|
|
测试代码呢 |
|
| 8分 |
貌似你的fwrite用法有问题。这个函数返回的是count（对应第三个参数），不是size（对应第二个参数），也就是每次成功都返回1（因为你给的count是1）。这个返回值不会有负数，如果失败，返回一个比count小的值。 你试试把第二个参数和第三个参数交换一下 |
|
日志文件通常不会太大，也不会写得很频繁，速度不是主要问题。但是处理异常是日志的主要功能之一，不能不考虑。突然断电的可能性不大，但是异常退出很常见。
分析速度之前先要确保每个用法的结果都是正确的。结果不一致时应先检查程序的正确性，结果不正确的情况下比较速度没有意义。 分析程序运行速度的程序，要使用大规模数据。10条记录的时间说明不了太大问题，起码要有上万条记录。 |
|
|
谢谢, 这个就解释了我在 16, 22, 23楼的疑问了, 非常感谢! |
|
|
重新测试了下速度:
Linux + no_flush: handle慢一个数量级, FILE与ofstream同一数量级, 但FILE略快 type: handle, speed: 365147 type: handle, speed: 553700 type: handle, speed: 620208 type: handle, speed: 660457 type: handle, speed: 683569 type: handle, speed: 698560 type: handle, speed: 710284 type: handle, speed: 718802 type: handle, speed: 726606 type: handle, speed: 729462 type: FILE, speed: 4557587 type: FILE, speed: 4805500 type: FILE, speed: 4266081 type: FILE, speed: 4493705 type: FILE, speed: 4859939 type: FILE, speed: 5003800 type: FILE, speed: 5072787 type: FILE, speed: 5132573 type: FILE, speed: 4857114 type: FILE, speed: 4878231 type: ofstream, speed: 3706144 type: ofstream, speed: 4089290 type: ofstream, speed: 4019945 type: ofstream, speed: 3723549 type: ofstream, speed: 3517367 type: ofstream, speed: 3674459 type: ofstream, speed: 3925795 type: ofstream, speed: 4079569 type: ofstream, speed: 3827552 type: ofstream, speed: 3762736 Linux + flush: handle慢四个数量级, FILE与ofstream同一数量级, 但FILE略快 type: handle, speed: 19 type: handle, speed: 29 type: handle, speed: 37 type: handle, speed: 37 type: handle, speed: 36 type: handle, speed: 36 type: handle, speed: 36 type: handle, speed: 37 type: handle, speed: 36 type: handle, speed: 38 type: FILE, speed: 627707 type: FILE, speed: 660472 type: FILE, speed: 670974 type: FILE, speed: 678343 type: FILE, speed: 683366 type: FILE, speed: 687264 type: FILE, speed: 690667 type: FILE, speed: 691783 type: FILE, speed: 692458 type: FILE, speed: 693900 type: ofstream, speed: 623825 type: ofstream, speed: 641605 type: ofstream, speed: 657575 type: ofstream, speed: 653685 type: ofstream, speed: 663389 type: ofstream, speed: 667315 type: ofstream, speed: 668244 type: ofstream, speed: 623023 type: ofstream, speed: 624355 type: ofstream, speed: 630020 Windows + no_flush: 三种方式, 速度都在同一个数量级, handle慢一些, 另外两个各有快慢; type: handle, speed: 71736 type: handle, speed: 91307 type: handle, speed: 99067 type: handle, speed: 102946 type: handle, speed: 105018 type: handle, speed: 106260 type: handle, speed: 107252 type: handle, speed: 108264 type: handle, speed: 108958 type: handle, speed: 109396 type: FILE, speed: 500532 type: FILE, speed: 334409 type: FILE, speed: 246716 type: FILE, speed: 228381 type: FILE, speed: 219697 type: FILE, speed: 213099 type: FILE, speed: 212147 type: FILE, speed: 209722 type: FILE, speed: 205950 type: FILE, speed: 205227 type: ofstream, speed: 435828 type: ofstream, speed: 313958 type: ofstream, speed: 297525 type: ofstream, speed: 274231 type: ofstream, speed: 296749 type: ofstream, speed: 312720 type: ofstream, speed: 330095 type: ofstream, speed: 332609 type: ofstream, speed: 322173 type: ofstream, speed: 345480 Windows + flush: handle慢四个数量级, FILE比ofstream快一半 type: handle, speed: 22 type: handle, speed: 37 type: handle, speed: 42 type: handle, speed: 44 type: handle, speed: 44 type: handle, speed: 42 type: handle, speed: 41 type: handle, speed: 41 type: handle, speed: 40 type: handle, speed: 40 type: FILE, speed: 121989 type: FILE, speed: 125874 type: FILE, speed: 126775 type: FILE, speed: 127286 type: FILE, speed: 127311 type: FILE, speed: 127464 type: FILE, speed: 127694 type: FILE, speed: 127870 type: FILE, speed: 128062 type: FILE, speed: 127990 type: ofstream, speed: 81132 type: ofstream, speed: 80382 type: ofstream, speed: 80439 type: ofstream, speed: 80652 type: ofstream, speed: 80746 type: ofstream, speed: 80823 type: ofstream, speed: 80845 type: ofstream, speed: 80889 type: ofstream, speed: 80810 type: ofstream, speed: 80738 自己总结下: FILE比ofstream略快, 比handle快很多, 但是handle是无二次缓冲的IO, 文件刷入更及时 |
|
|
好了 谢谢各位 结贴了
|
|
|
仅供参考
#include <sys\stat.h>
#include <io.h>
#include <fcntl.h>
#include <share.h>
#include <stdlib.h>
#include <stdio.h>
#include <conio.h>
#include <string.h>
#define MAX_CLU_BYTES 65536
FILE *fo;
int fh;
__int64 offs,offs1;
__int64 rvi64;
int rv,wrv;
unsigned char buf[MAX_CLU_BYTES];
char ofn[_MAX_PATH];
char offstr[80];
void strcpybutcomma(char *t,char *s) {
char c;
while (1) {
c = *s++;
if ("",""!=c) *t++ = c;
if (0==c) break;
}
}
void main(int argc,char **argv) {
if (argc<3) {
printf("Copy File Tail.\n");
printf("Usage:\n");
printf(" cft filename.ext offset_begin[-offset_end]\n");
printf("Copy filename.ext offset_begin[-offset_end] to offset_begin[-offset_end]-filename.ext\n");
printf("Note: Byte at offset_end is NOT included.\n");
printf("Example:\n");
printf(" cft abc.rar 12345\n");
printf("Copy abc.rar offset 12345-end to 12345-abc.rar\n");
printf(" cft abc.rar 123-12345\n");
printf("Copy abc.rar offset 123-12345 to 123-12345-abc.rar\n");
printf(" cft abc.rar 0xAB-0xCD\n");
printf("Copy abc.rar offset 0xAB-0xCD to 0xAB-0xCD-abc.rar\n");
return;
}
strcpybutcomma(offstr,argv[2]);
rv=sscanf(offstr,"%I64i-%I64i",&offs,&offs1);
if (rv==0) {
printf("offset %s is not number\n",argv[2]);
return;
}
fh=_sopen(argv[1],_O_BINARY|_O_RDONLY|_O_RANDOM,_SH_DENYWR);
if (fh==-1) {
printf("_sopen %s errno=%d\n",argv[1],errno);
return;
}
if (rv==1) {
offs1=_filelengthi64(fh);
if (offs1==-1i64) {
printf("%I64=_filelengthi64 errno=%d\n",offs1,errno);
_close(fh);
return;
}
} else {//rv==2
if (offs1<offs) {
printf("%s offset_begin>offset_end error\n",argv[2]);
_close(fh);
return;
}
}
rvi64=_lseeki64(fh,offs,SEEK_SET);
if (rvi64!=offs) {
printf("%I64u=_lseeki64 %I64u errno=%d\n",rvi64,offs,errno);
_close(fh);
return;
}
sprintf(ofn,"%s-",offstr);
strcat(ofn,argv[1]);
fo=fopen(ofn,"wb");
if (fo==NULL) {
_close(fh);
printf("fopen %s error\n",ofn);
return;
}
cprintf("\n%I64u\r",offs);
while (1) {
rv=_read(fh,buf,(unsigned int)__min(offs1-offs,MAX_CLU_BYTES));
if (rv==0) break;//
if (rv<0) {
fclose(fo);
_close(fh);
printf("_read %s offset %I64u error\n",argv[1],offs);
return;
}
wrv=fwrite(buf,1,rv,fo);
if (wrv!=rv) {
fclose(fo);
_close(fh);
printf("fwrite %s error\n",ofn);
return;
} else {
offs+=rv;
cprintf("%I64u\r",offs);
if (offs>=offs1) break;//
}
}
fclose(fo);
_close(fh);
printf("Copy %s offset %s to %s OK.\n",argv[1],argv[2],ofn);
}
|
|
|
Windows下可以试试直接调用Win API：
WriteFile WriteFileEx |
|
|
直接用api估计更悲剧 |
|
|
不可能。因为在Windows下， ofstream::write() ， fwrite() ， write()最终都调用的是Win API。 再参考C:\Program Files\Microsoft Visual Studio 10.0\VC\crt\src\write.c |
|
|
所以才可能悲剧，这里面主要是系统API之上的各种缓存造成的，可以看看apue，虽然不是windows，但是原理是一样的 |
|
|
我以为你说“悲剧”是指Win API速度会比其它方案慢呢。
|
|
|
感谢楼上两位的回复.
|
|
|
你这完全是为了提高编程能力啊
|
|
|
我自己亲自经历过C++ iostream比C文件读写慢好多的情况。写文件操作绝对c文件读写效率高。
|
|
CodeBye 版权所有丨如未注明 , 均为原创丨本网站采用BY-NC-SA协议进行授权 , 转载请注明[经历] write+commit与ofstream的比较！
![[小白求帮助]分配的内存什么时候需要手动释放](https://www.codebye.com/wp-content/themes/douth/timthumb.php?src=https://www.codebye.com/wp-content/themes/douth/assets/img/pic/3.jpg&h=110&w=185&q=90&zc=1&ct=1)