EveryNcallback buffer overflow on 6321 device

[RobHettema]

I need some help.

On machines of ours we use the 6321 PCIe-6321 multifunctional I/O device.
They have been running fine for a few years, but since about 1 year ago we ran into problems.
We assume the problems we have are related to the fact that a customer application is running on the same PC and interferes with our application.
This client performs web-access to a central server in Sweden.
It seems the problems are more severe the further away a machine is from sweden.
We have machines in sweden, holland, france and brasil. And Sweden does not report the problem but the other locations do.

So what is the the actual problem :

2020-11-23 06:13:03,86-DAQmx callback error: Attempted to read samples that are no longer available.
The requested sample was previously available, but has since been overwritten.
Increasing the buffer size, reading the data more frequently, or specifying a fixed number of samples
to read instead of reading all available samples might correct the problem.
Property: DAQmx_Read_RelativeTo Requested Value: DAQmx_Val_CurrReadPos
Property: DAQmx_Read_Offset
Requested Value: 0
Device: Dev2
Task Name: _unnamedTask<3>
Status Code: -200279

Now the source code I am using.
The appliction is not a Labview application, but a c++ application build with Embarcadero.
we also have a PCI-6518 Digital IO card in the system, so DeviceNr points to the correct board
and at startup AnOut0 and 1 are filled with the correct settings.
If the 6321 is device1 :
AnOut0 = "Dev1/ao0";
AnOut1 = "Dev1/ao1";

Initialisation of my tasks:
(DWprintf writes debug information to a seperate window. The error description above is taken from that window)

void __fastcall TAnaThread::StartRTdata(int cycletime, int NrItems)
{
DWprintf("Start RT data called");

/*********************************************/
// DAQmx Configure Code
/*********************************************/
if (AnOutputs == 0)
{
// analog output task :
DAQmxErrChk (DAQmxCreateTask("",&AnOutputs));
DAQmxErrChk (DAQmxCreateAOVoltageChan(AnOutputs, AnOut0.c_str(), "", -10.0,10.0, DAQmx_Val_Volts, ""));
DAQmxErrChk (DAQmxCreateAOVoltageChan(AnOutputs, AnOut1.c_str(), "", -10.0,10.0, DAQmx_Val_Volts, ""));
}
if (AnIns == 0)
{
// analog input task :
DAQmxErrChk (DAQmxCreateTask("",&AnIns));
if (DeviceNr == 1)
{
DAQmxErrChk (DAQmxCreateAIVoltageChan(AnIns, "Dev1/ai0:5", "", DAQmx_Val_Diff ,-10.0,10.0,DAQmx_Val_Volts,NULL));
DWprintf("Analog input task created for dev1");
}
else
{
DAQmxErrChk (DAQmxCreateAIVoltageChan(AnIns, "Dev2/ai0:5", "", DAQmx_Val_Diff ,-10.0,10.0,DAQmx_Val_Volts,NULL));
DWprintf("Analog input task created for dev2");
}
DAQmxErrChk (DAQmxCfgSampClkTiming(AnIns, "", 100, DAQmx_Val_Rising, DAQmx_Val_ContSamps, 100));
DAQmxErrChk (DAQmxRegisterEveryNSamplesEvent(AnIns, DAQmx_Val_Acquired_Into_Buffer, 10, 0, EveryNCallback, NULL));
DAQmxErrChk (DAQmxRegisterDoneEvent(AnIns, 0, DoneCallback, NULL));
}
if (Freq1 == 0)
{
// timer tasks :
DAQmxErrChk (DAQmxCreateTask("",&Freq1));
if (DeviceNr == 1)
{
DAQmxErrChk (DAQmxCreateCIFreqChan(Freq1,"Dev1/ctr0","", 0.1, 1000.000000, DAQmx_Val_Hz, DAQmx_Val_Rising, DAQmx_Val_LowFreq1Ctr, 0.001, 4, ""));
DWprintf("Freq1 task created for dev1");
}
else
{
DAQmxErrChk (DAQmxCreateCIFreqChan(Freq1,"Dev2/ctr0","", 0.1, 1000.000000, DAQmx_Val_Hz, DAQmx_Val_Rising, DAQmx_Val_LowFreq1Ctr, 0.001, 4, ""));
DWprintf("Freq1 task created for dev2");
}
DAQmxErrChk (DAQmxCfgImplicitTiming(Freq1, DAQmx_Val_ContSamps, 4000));
}
if (Freq2 == 0)
{
DAQmxErrChk (DAQmxCreateTask("",&Freq2));
if (DeviceNr == 1)
{
DAQmxErrChk (DAQmxCreateCIFreqChan(Freq2,"Dev1/ctr1","", 0.1, 1000.000000, DAQmx_Val_Hz, DAQmx_Val_Rising, DAQmx_Val_LowFreq1Ctr, 0.001, 4, ""));
DWprintf("Freq2 task created for dev1");
}
else
{
DAQmxErrChk (DAQmxCreateCIFreqChan(Freq2,"Dev2/ctr1","", 0.1, 1000.000000, DAQmx_Val_Hz, DAQmx_Val_Rising, DAQmx_Val_LowFreq1Ctr, 0.001, 4, ""));
DWprintf("Freq2 task created for dev2");
}
DAQmxErrChk (DAQmxCfgImplicitTiming(Freq2, DAQmx_Val_ContSamps, 4000));
}
/*********************************************/
// DAQmx Start Tasks
/*********************************************/
DAQmxErrChk (DAQmxStartTask(AnOutputs));
DAQmxErrChk (DAQmxStartTask(AnIns));
DAQmxErrChk (DAQmxStartTask(Freq1));
DAQmxErrChk (DAQmxStartTask(Freq2));

XmitSetpoint(0, 0.0);
XmitSetpoint(1, 0.0);
RTdataNrItems = NrItems;
RTdataTimer->Interval = 100; // cycletime;
RTdataTimer->Enabled = true;
return;
Error:
if( DAQmxFailed(error) )
{
DAQmxGetExtendedErrorInfo(errBuff,2048);
DWprintf("DAQmx Error Start RTdata: %d %s\n",tel, errBuff);
}
}

The actual error occurs in my callback routine :

int32 CVICALLBACK EveryNCallback(TaskHandle taskHandle, int32 everyNsamplesEventType, uInt32 nSamples, void *callbackData)
{
int32 error=0;
char errBuff[2048]={'\0'};
static int totalRead=0;
int32 read=0;
float64 data[2000];
int tel;
float64 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5 = 0.0;

/*********************************************/
// DAQmx Read Code
/*********************************************/
DAQmxErrChk (DAQmxReadAnalogF64(taskHandle, 10, 10.0, DAQmx_Val_GroupByChannel, data, 2000, &read, NULL));
if( read>0 )
{
tmp0 = 0.0;
tmp1 = 0.0;
tmp2 = 0.0;
tmp3 = 0.0;
tmp4 = 0.0;
tmp5 = 0.0;
for (tel = 0 ; tel < read; tel++)
{
tmp0 = tmp0 + data[tel];
tmp1 = tmp1 + data[tel + read];
tmp2 = tmp2 + data[tel + (read * 2)];
tmp3 = tmp3 + data[tel + (read * 3)];
tmp4 = tmp4 + data[tel + (read * 4)];
tmp5 = tmp5 + data[tel + (read * 5)];
}
AnIn0 = (AnIn0 * 0.7) + ((tmp0 / read) * 0.3);
AnIn1 = (AnIn1 * 0.7) + ((tmp1 / read) * 0.3);
AnIn2 = (tmp2 / read); // (AnIn2 * 0.7) + ((tmp2 / read) * 0.3);
AnIn3 = (tmp3 / read); // (AnIn3 * 0.7) + ((tmp3 / read) * 0.3);
AnIn4 = (tmp4 / read); // (AnIn4 * 0.7) + ((tmp4 / read) * 0.3);
AnIn5 = (tmp5 / read); // (AnIn5 * 0.7) + ((tmp5 / read) * 0.3);
if (LogData)
{
LogData0[LogPointer] = FreqIn0;
LogData1[LogPointer] = FreqIn1;
LogData2[LogPointer] = AnIn0; // data[tel];
LogData3[LogPointer] = AnIn1; // data[tel + read];
LogPointer++;
if (LogPointer >= NrOfSamples)
{
LogData = false;
}
}
}

Error:
if( DAQmxFailed(error) )
{
DAQmxGetExtendedErrorInfo(errBuff,2048);
/*********************************************/
// DAQmx Stop Code
/*********************************************/
DAQmxStopTask(taskHandle);
DAQmxClearTask(taskHandle);
DWprintf("DAQmx callback error: %s\n",errBuff);
}
return 0;
}

I have no control over what the other application is doing, so I am trying to find a fix in our application.
The only task that gives the error is task 2. I have not seen it for task 1, 3 and 4.

Any help/suggestion/workaround is welcome in finding a solution for keeping the tasks running (even with sometimes a buffer overflow).
If I can keep running the tasks (currently the Error section stops the task that reports the problem) and have valid data i am happy.

Thanks in advance for any help.

Rob

[Kevin_Price]

First, I'd say that there's no *certainty* of fixing things in your application without understanding the real cause of the problem at some locations and not others.

Second, I don't know the details of any text language API for DAQmx, but it appears to me that your AI task:

- samples at 100 Hz

- *attempts* to set a 100 sample buffer size (which will actually be coerced to 1000)

- sets up a callback to be fired off for every 10 samples acquired into the buffer

- on every callback, you read 10 samples from the buffer.

All of these things seem quite reasonable.  You only fire the callback 10 times a second and you end up with a 10 second buffer (despite asking for only 1).  HOWEVER, I was very recently in a thread where a poster reported that callbacks were occasionally missed.  I suggested a workaround and perhaps you'd want to consider it too. 

    It will only help if the problem arises due to an *accumulation* of many missed callbacks that make you gradually build up more and more backlog in the buffer.  It won't help if one single event causes *many* consecutive callbacks to be missed for some reason, and the buffer overflow happens all at once.

There's a whole different approach that would prevent the fatal errors, but you wouldn't acquire contiguous data if any callbacks were missed, and the working theory at the moment is that some might be.

   I have no idea what the text language syntax would be, but in LabVIEW there's a "DAQmx Read property node" that allows you to configure various properties related to retrieving data from input tasks.  There are 3 properties that would matter:

- set OverwriteMode to "overwrite unread samples".  This prevents the fatal error when the buffer overflows.  It also can allow you have a gap in your data without realizing it.

- set RelativeTo to "most recent sample"

- set Offset to "-10"

   Then on each callback when you read 10 samples, the driver would start from a location 10 samples ago and give you the 10 that catch you up to "right now".  If ever a callback were missed such that the next one found you with 20 unread samples in the buffer, the earliest 10 would be ignored, the most recent 10 would be returned, and you'd have non-contiguous data.

There are variations of both these options that are based on polling rather than relying on the callback mechanism.

Meanwhile, RAM is pretty cheap, so bumping the buffer up to 10k samples couldn't hurt. I'd at least couple that with my linked workaround though.

-Kevin P

[RobHettema]

Thank you Kevin,

Since I have to test the changes I make remotely I will start with just increasing the buffer size.

I have increased the buffer now to 30000 for my analog inputs task.:

if (AnIns == 0)
{
// analog input task :
DAQmxErrChk (DAQmxCreateTask("",&AnIns));
DAQmxCfgInputBuffer (AnIns, 30000); 
if (DeviceNr == 1)
{
DAQmxErrChk (DAQmxCreateAIVoltageChan(AnIns, "Dev1/ai0:5", "", DAQmx_Val_Diff ,-10.0,10.0,DAQmx_Val_Volts,NULL));
DWprintf("Analog input task created for dev1");
}
else
{
DAQmxErrChk (DAQmxCreateAIVoltageChan(AnIns, "Dev2/ai0:5", "", DAQmx_Val_Diff ,-10.0,10.0,DAQmx_Val_Volts,NULL));
}
DAQmxErrChk (DAQmxCfgSampClkTiming(AnIns, "", 100, DAQmx_Val_Rising, DAQmx_Val_ContSamps, 100));
DAQmxErrChk (DAQmxRegisterEveryNSamplesEvent(AnIns, DAQmx_Val_Acquired_Into_Buffer, 10, 0, EveryNCallback, NULL));
DAQmxErrChk (DAQmxRegisterDoneEvent(AnIns, 0, DoneCallback, NULL));
DAQmxErrChk (DAQmxStartTask(AnIns));
}

And for my frequency tasks as well :

if (Freq1 == 0)
{
// timer tasks :
DAQmxErrChk (DAQmxCreateTask("",&Freq1));
if (DeviceNr == 1)
{
DAQmxErrChk (DAQmxCreateCIFreqChan(Freq1,"Dev1/ctr0","", 0.1, 1000.000000, DAQmx_Val_Hz, DAQmx_Val_Rising, DAQmx_Val_LowFreq1Ctr, 0.001, 4, ""));
DWprintf("Freq1 task created for dev1");
}
else
{
DAQmxErrChk (DAQmxCreateCIFreqChan(Freq1,"Dev2/ctr0","", 0.1, 1000.000000, DAQmx_Val_Hz, DAQmx_Val_Rising, DAQmx_Val_LowFreq1Ctr, 0.001, 4, ""));
DWprintf("Freq1 task created for dev2");
}
DAQmxErrChk (DAQmxCfgImplicitTiming(Freq1, DAQmx_Val_ContSamps, 30000));
DAQmxErrChk (DAQmxStartTask(Freq1));
}

While this program is being tested on 1 machine I will look into these properties you mentioned. They look promising.

For my application it is not a big problem that I might loose a few samples..

-Rob

[RobHettema]

OH,

I think I made a small mistake..:

I have set the buffer top 30000

DAQmxCfgInputBuffer (AnIns, 30000);

But later I set it to 100 (=1000).

DAQmxErrChk (DAQmxCfgSampClkTiming(AnIns, "", 100, DAQmx_Val_Rising, DAQmx_Val_ContSamps, 100));

I think i have to do

DAQmxErrChk (DAQmxCfgSampClkTiming(AnIns, "", 100, DAQmx_Val_Rising, DAQmx_Val_ContSamps, 30000)); and forget about the  DAQmxCfgInputBuffer (AnIns, 30000);

[Kevin_Price]

Your code correction for buffer sizing sounds like the right way to go.   Over here in LabVIEW land, I've occasionally called the buffer sizing function explicitly.  Any time I did it though it was always *AFTER* the call to set up DAQmx Timing.  I'm honestly unsure what you'd end up with when calling those functions in the opposite order like you did.

Mostly I did it many years ago, before I knew the rule DAQmx used to set a minimum default buffer size for continuous acquistion.  I think I was uncertain whether it might give me a *smaller* buffer than requested, so I called the explicit sizing function to make sure.  Any more, I don't find myself calling the explicit sizing function.  The buffer size for continuous acquisition generally doesn't need to be precise, merely "big enough" which I now know how to accomplish reliably in the call to DAQmx Timing.  Just like you're gonna do now too.

FWIW, your 10 second buffer was already, generally speaking, REALLY big for the  task at hand.  Dunno the root cause of your buffer overflow problem, but suspect some unique config or circumstances.

However, merely making the buffer bigger is probably only going to delay the problem.  You really should try the first workaround I linked to.  (And if you do you probably shouldn't need the bigger buffer, though it also shouldn't hurt any.)   The query for # of samples available also uses a DAQmx Read property node over here in LabVIEW.  Hope that helps you track it down in your API.

-Kevin P

[RobHettema]

I have added the properties and it looks promising.

The situation was about 10 - 15 times per day the callback error, and until now ( 2 hours production) no problem yet.

Let see how the graveyard shift will do.

[RobHettema]

After about 1 day production on 2 machines i have not seen the callback error anymore.

But another problem sometimes pops up.

My analog input task runs all the time and when a production test is running the measured analog values are shown on the operator screen in vertical bars.

The measured values represent brakeforce on a left and right wheel of a vehicle.

The problem reported to me is that sometimes the bars behave "funny". Sometimes they freeze, sometimes they seem to react not at the correct moment and sometimes the vertical bars show brakeforce that varies even without touching the brakepedal in the vehicle at all.

This gives me the idea that the callback routine sometimes reads data from the buffer that is not the last acquired data.

Something like the read-pointer in the buffer is not synchronized with the write-pointer anymore, but is running behind in time.

@Kevin : In your thread about the work around the final solution that KnivarLarsson choose was not to use the callback completely.

That might do the trick for me as well.

Also your suggestion to check how much data is available in the buffer after reading and to do an extra read action to catch up might help.

But an other idea i have is to force the read pointer to the most recent write position by adding this just before I read the samples :

DAQmxSetReadRelativeTo(AnIns, DAQmx_Val_MostRecentSamp);

DAQmxSetReadOffset(AnIns, -10);

But I do not know if this will "synchronize" the read-pointer of samples again (and thus just skip unread samples and move to the last sampled data) or if it is just needed where the task is created.

// Rob

[Kevin_Price]

Yeah, the normal behavior under DAQmx is that the read pointer (pointing just past the most recent sample you read out of the buffer from the app side) is distinct and independent of the write pointer (pointing just past the most recent sample the driver delivered into the buffer from the device).   Each new read starts from the position where the read pointer had left off, regardless of how much data's been written into the buffer in the meantime.

It isn't yet clear how or why your reads would fall behind the writes -- your DAQmx code looked nominally correct and wasn't particularly demanding of the hardware or system.  Nevertheless, once they *do* fall behind the writes, your original approach wouldn't ever catch up b/c it was trying to read the same # of samples that the driver would have newly written.  And with each missed callback or read opportunity, you'd fall behind another 10 samples.  Eventually, you could fall behind far enough that the next write by the driver would make the write pointer "lap" the read pointer, triggering the buffer overflow error you were occasionally getting.

When the live graph seems to be showing data from the past, that's a sign of a backlog somewhere.  Very possibly in servicing the DAQmx buffer, but there could be other bottlenecks, depending on your app.  The "freezing" behavior sounds like it could be an overwhelmed CPU, which in turn could be a root cause for the app's DAQmx reads to fall behind the driver's writes.

The code you listed at the end involving "RelativeTo" and "Offset" look like exactly what I was driving at in the 2nd approach back in msg #2.  But you should probably also explicitly set the property that allows unread samples to be overwritten (without throwing that overflow error).

   Yeah, that should do exactly what you want.  The "MostRecentSample" is another name for the write pointer position.  Your code will force the non-default behavior of starting the read at a position -10 samples from where the driver will write the next samples it delivers to the buffer.  In other words, you can request 10 samples, and you'll immediately retrieve the 10 most recent ones already delivered by the driver.  You'd skip right over any unread samples.

-Kevin P

[RobHettema]

OK, I will try these new idea's and also I will do a complete device reset before I start a new test:

if (DeviceNr == 1)
{
DAQmxErrChk (DAQmxResetDevice("dev1")); // Fresh start ! Kills all actions (read/write) and clears all tasks.
DWprintf("Reset device 1");
}
else
{
DAQmxErrChk (DAQmxResetDevice("dev2"));
DWprintf("Reset device 2");
}
Sleep(500); // give device some time to re-initialize

I still have to look for a better way to see if the reset is finished.

I will (timer based) read data 10 times per second and force the pointer to the correct position :

In my timer routine I will do :

if (AnIns != 0)
{
error = DAQmxSetReadOverWrite(AnIns, DAQmx_Val_OverwriteUnreadSamps); // allow overwrite of buffer
if( DAQmxFailed(error) )
{
DAQmxGetExtendedErrorInfo(errBuff,2048);
DWprintf("DAQmx Overwrite samples error: %s\n",errBuff);
}
error = DAQmxSetReadRelativeTo(AnIns, DAQmx_Val_MostRecentSamp); // set read pointer to most recent
if( DAQmxFailed(error) )
{
DAQmxGetExtendedErrorInfo(errBuff,2048);
DWprintf("DAQmx Set read relative error: %s\n",errBuff);
}
error = DAQmxSetReadOffset(AnIns, -10); // read previous 10 samples to get last 100 millisecond data
if( DAQmxFailed(error) )
{
DAQmxGetExtendedErrorInfo(errBuff,2048);
DWprintf("DAQmx Set read offset error: %s\n",errBuff);
}

error = DAQmxReadAnalogF64(AnIns, 10, 10.0, DAQmx_Val_GroupByChannel, data, 2000, &read, NULL);
if( DAQmxFailed(error) )
{
DAQmxGetExtendedErrorInfo(errBuff,2048);
DWprintf("DAQmx read error analog input buffer: %s\n",errBuff);
}
else
{
if( read>0 )
{

.... 

[Kevin_Price]

The full device reset shouldn't be necessary.  In the long run, whatever situation gets resolved by doing it should be solved more "gently".

You won't need to do *all* that stuff every iteration, though it also shouldn't be an issue when only calling the routine 10 times a second.

    Except the call that allows DAQmx to OverwriteUnreadSamples.  I only ever make that call 1 time and I do it *before* the task is started.  I *think* I learned long ago that it's necessary to do it that way, but I couldn't swear to it.  For sure it does work right when I do it that way though.

I've also learned from experience that the settings for RelativeTo and Offset are *persistent*.  In situations like yours, I would typically also set RelativeTo just once.  For the sake of some flexibility, I would generally have my code set Offset every iteration just to support the possibility of changing the # samples I read each iteration.desired.

   One other thing I've learned to do is this:  after starting the task but before setting the RelativeTo property to MostRecentSamp, I do a single Read of the standard # of samples.  I ignore the data, but doing this ensures that when I start the calls that retrieve recent past-tense data, there's sure to be enough past samples built up already.

-Kevin P