cppcolors | Embarcadero RAD Studio Delphi C++Builder Blogs

Author: h.mohri

Convert from CSV file to “JSON file” of “TFDMemTable format”.

This program uses C++Builder 10.2 Tokyo Release 1.

include

<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><iostream></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><fstream></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><string></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><codecvt></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><FireDAC.Comp.Client.hpp></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><FireDAC.Stan.StorageJSON.hpp></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><FireDAC.Stan.Intf.hpp></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><memory></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><vector></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><sstream></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><mutex></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><thread></span>
<span class="synPreProc" style="color:#9355e6;">#include </span><span class="synConstant" style="color:#ff6666;"><chrono></span>

#include

#include <FireDAC.Comp.Client.hpp>

#include <FireDAC.Stan.StorageJSON.hpp>

#include <FireDAC.Stan.Intf.hpp>

#include

code

<span class="synType" style="color:#3ec63e;">struct</span> _Tcsv_to_memtable
{
    std::mutex mx_;
    std::vector<std::thread> vth_;
    std::unique_ptr<TFDMemTable> mem_table_{std::make_unique<TFDMemTable>(<span class="synConstant" style="color:#ff6666;">nullptr</span>)};
    std::unique_ptr<TFDStanStorageJSONLink> jl_{std::make_unique<TFDStanStorageJSONLink>(<span class="synConstant" style="color:#ff6666;">nullptr</span>)};
    <span class="synType" style="color:#3ec63e;">void</span> create_column(std::vector<String>& v)
    {
        <span class="synComment" style="color:#4f80e5;">//Field creation of "TFDMemTable"</span>
        <span class="synComment" style="color:#4f80e5;">//The field name is the first line of CSV File</span>
        <span class="synStatement" style="color:#d88a17;">for</span> (<span class="synType" style="color:#3ec63e;">auto</span> s: v)
        {
            <span class="synType" style="color:#3ec63e;">auto</span> fld = mem_table_->FieldDefs->AddFieldDef();
            <span class="synComment" style="color:#4f80e5;">//Discard the "BOM (Byte Order Mark)" if present.</span>
            fld->Name = StringReplace(s, <span class="synSpecial" style="color:#c000c0;">L'x0feff'</span>, <span class="synConstant" style="color:#ff6666;">""</span>, TReplaceFlags());
            fld->DataType = TFieldType::ftWideString;
        }
        mem_table_->Active = <span class="synConstant" style="color:#ff6666;">true</span>;
    }
    <span class="synType" style="color:#3ec63e;">void</span> create_line(std::vector<String>& v)
    {
        <span class="synComment" style="color:#4f80e5;">//Add 1 line of "CSV File" to "TFDMemTable"</span>
        <span class="synType" style="color:#3ec63e;">int</span> i{<span class="synConstant" style="color:#ff6666;">0</span>};
        mem_table_->Append();
        <span class="synStatement" style="color:#d88a17;">for</span> (<span class="synType" style="color:#3ec63e;">auto</span> s: v)
        {
            <span class="synComment" style="color:#4f80e5;">//One item for one field.</span>
            mem_table_->Fields-><span class="synStatement" style="color:#d88a17;">operator</span>[](i)->AsWideString = s;
            ++i;
        }
        mem_table_->Post();
    }
    <span class="synType" style="color:#3ec63e;">void</span> create_line(std::wstring ws, <span class="synType" style="color:#3ec63e;">int</span> line_num)
    {
        std::lock_guard<std::mutex> lk(mx_);
        <span class="synStatement" style="color:#d88a17;">if</span> (ws.length() > <span class="synConstant" style="color:#ff6666;">0</span>)
        {
            <span class="synComment" style="color:#4f80e5;">//Inserted by dividing one line of CSV File to vector<String>.</span>
            std::vector<String> v1;
            std::wstring column_data;
            std::wistringstream line_data(ws);
            <span class="synStatement" style="color:#d88a17;">while</span> (std::getline(line_data, column_data, <span class="synConstant" style="color:#ff6666;">L','</span>))
            {
                v1.push_back(column_data.c_str());
            }
            <span class="synStatement" style="color:#d88a17;">switch</span> (line_num)
            {
            <span class="synStatement" style="color:#d88a17;">case</span> <span class="synConstant" style="color:#ff6666;">0</span>:
                create_column(v1);
                <span class="synStatement" style="color:#d88a17;">break</span>;
            <span class="synStatement" style="color:#d88a17;">default</span>:
                create_line(v1);
            }
        }

    }
    <span class="synType" style="color:#3ec63e;">void</span> threads_join()
    {
        <span class="synComment" style="color:#4f80e5;">//Wait for the termination of all threads.</span>
        <span class="synStatement" style="color:#d88a17;">for</span> (<span class="synType" style="color:#3ec63e;">auto</span>& th: vth_)
        {
            th.join();
        }
    }

};
<span class="synType" style="color:#3ec63e;">int</span> _tmain(<span class="synType" style="color:#3ec63e;">int</span> argc, _TCHAR* argv[])
{
    std::chrono::system_clock::time_point now = std::chrono::system_clock::now();
    _Tcsv_to_memtable memtable_;
    UnicodeString argv_     = argv[<span class="synConstant" style="color: #ff6666;">1</span>];  <span class="synComment" style="color:#4f80e5;">//Specify CSV Fire for input.</span>
    UnicodeString arg_o_    = argv[<span class="synConstant" style="color: #ff6666;">2</span>];  <span class="synComment" style="color:#4f80e5;">//JSON output destination specification made with "TFDMemTable".</span>
    <span class="synStatement" style="color:#d88a17;">if</span> (FileExists(argv_))
    {
        std::wfstream file_;
        file_.open(argv_.w_str(), std::ios::in);
        <span class="synComment" style="color:#4f80e5;">//The CSV file is utf8 format.</span>
        file_.imbue(std::locale(std::locale::empty(), <span class="synStatement" style="color:#d88a17;">new</span> std::codecvt_utf8<<span class="synType" style="color:#3ec63e;">wchar_t</span>>));

        std::wstring buff;
        <span class="synType" style="color:#3ec63e;">int</span> line_num_{<span class="synConstant" style="color:#ff6666;">0</span>};
        <span class="synStatement" style="color:#d88a17;">while</span> (std::getline(file_, buff))
        {
            <span class="synComment" style="color:#4f80e5;">//The append processing to "TFDMemTable" threads line by line.</span>
            memtable_.vth_.push_back(std::thread([&memtable_, &buff, line_num_](){
                memtable_.create_line(buff, line_num_);
                }));
            <span class="synComment" style="color:#4f80e5;">//Since the command line is multibyte, AnsiString conversion is done.</span>
            AnsiString as_ = buff.c_str();
            std::cout << as_ << std::endl;
            ++line_num_;
        }

        memtable_.threads_join();
        <span class="synComment" style="color:#4f80e5;">//Finally, save the TFDMemTable contents as a JSON file.</span>
        memtable_.mem_table_->SaveToFile(arg_o_, TFDStorageFormat::sfJSON);

    }
    <span class="synType" style="color:#3ec63e;">auto</span> keika =  std::chrono::system_clock::now() - now;
    std::cout << std::chrono::duration_cast<std::chrono::milliseconds>(keika).count();
    std::cout << <span class="synConstant" style="color:#ff6666;">" millisecond."</span> << std::endl ;
    <span class="synStatement" style="color:#d88a17;">return</span> <span class="synConstant" style="color:#ff6666;">0</span>;
}

100

101

102

103

struct _Tcsv_to_memtable

{

std::mutex mx_;

std::vector<std::thread> vth_;

std::unique_ptr mem_table_{std::make_unique(nullptr)};

std::unique_ptr jl_{std::make_unique(nullptr)};

void create_column(std::vector& v)

{

">//Field creation of "TFDMemTable"

synComment" style="color:#4f80e5;">//The field name is the first line of CSV File

synStatement" style="color:#d88a17;">for (synType" style="color:#3ec63e;">auto s: v)

{

synType" style="color:#3ec63e;">auto fld = mem_table_->FieldDefs->AddFieldDef();

synComment" style="color:#4f80e5;">//Discard the "BOM (Byte Order Mark)" if present.

fld->Name = StringReplace(s, synSpecial" style="color:#c000c0;">L'x0feff', synConstant" style="color:#ff6666;">"", TReplaceFlags());

fld->DataType = TFieldType::ftWideString;

}

mem_table_->Active = synConstant" style="color:#ff6666;">true;

}

synType" style="color:#3ec63e;">void create_line(std::vector& v)

{

synComment" style="color:#4f80e5;">//Add 1 line of "CSV File" to "TFDMemTable"

synType" style="color:#3ec63e;">int i{synConstant" style="color:#ff6666;">0};

mem_table_->Append();

synStatement" style="color:#d88a17;">for (synType" style="color:#3ec63e;">auto s: v)

{

synComment" style="color:#4f80e5;">//One item for one field.

mem_table_->Fields->synStatement" style="color:#d88a17;">operator[](i)->AsWideString = s;

++i;

}

mem_table_->Post();

}

synType" style="color:#3ec63e;">void create_line(std::wstring ws, synType" style="color:#3ec63e;">int line_num)

{

std::lock_guard lk(mx_);

synStatement" style="color:#d88a17;">if (ws.length() > synConstant" style="color:#ff6666;">0)

{

synComment" style="color:#4f80e5;">//Inserted by dividing one line of CSV File to vector.

std::vector v1;

std::wstring column_data;

std::wistringstream line_data(ws);

synStatement" style="color:#d88a17;">while (std::getline(line_data, column_data, synConstant" style="color:#ff6666;">L','))

{

v1.push_back(column_data.c_str());

}

synStatement" style="color:#d88a17;">switch (line_num)

{

synStatement" style="color:#d88a17;">case synConstant" style="color:#ff6666;">0:

create_column(v1);

synStatement" style="color:#d88a17;">break;

synStatement" style="color:#d88a17;">default:

create_line(v1);

}

synType" style="color:#3ec63e;">void threads_join()

{

synComment" style="color:#4f80e5;">//Wait for the termination of all threads.

synStatement" style="color:#d88a17;">for (synType" style="color:#3ec63e;">auto& th: vth_)

{

th.join();

}

};

synType" style="color:#3ec63e;">int _tmain(synType" style="color:#3ec63e;">int argc, _TCHAR* argv[])

{

std::chrono::system_clock::time_point now = std::chrono::system_clock::now();

_Tcsv_to_memtable memtable_;

UnicodeString argv_ = argv[synConstant" style="color: #ff6666;">1]; synComment" style="color:#4f80e5;">//Specify CSV Fire for input.

UnicodeString arg_o_ = argv[synConstant" style="color: #ff6666;">2]; synComment" style="color:#4f80e5;">//JSON output destination specification made with "TFDMemTable".

synStatement" style="color:#d88a17;">if (FileExists(argv_))

{

std::wfstream file_;

file_.open(argv_.w_str(), std::ios::in);

synComment" style="color:#4f80e5;">//The CSV file is utf8 format.

file_.imbue(std::locale(std::locale::empty(), synStatement" style="color:#d88a17;">new std::codecvt_utf8<synType" style="color:#3ec63e;">wchar_t>));

std::wstring buff;

synType" style="color:#3ec63e;">int line_num_{synConstant" style="color:#ff6666;">0};

synStatement" style="color:#d88a17;">while (std::getline(file_, buff))

{

synComment" style="color:#4f80e5;">//The append processing to "TFDMemTable" threads line by line.

memtable_.vth_.push_back(std::thread([&memtable_, &buff, line_num_](){

memtable_.create_line(buff, line_num_);

}));

synComment" style="color:#4f80e5;">//Since the command line is multibyte, AnsiString conversion is done.

AnsiString as_ = buff.c_str();

std::cout << as_ << std::endl;

++line_num_;

}

memtable_.threads_join();

synComment" style="color:#4f80e5;">//Finally, save the TFDMemTable contents as a JSON file.

memtable_.mem_table_->SaveToFile(arg_o_, TFDStorageFormat::sfJSON);

}

synType" style="color:#3ec63e;">auto keika = std::chrono::system_clock::now() - now;

std::cout << std::chrono::duration_cast(keika).count();

std::cout << synConstant" style="color:#ff6666;">" millisecond." << std::endl ;

synStatement" style="color:#d88a17;">return synConstant" style="color:#ff6666;">0;

}

There is a project file in github, you can try it right away.
GitHub – mojeld/csvfstream_to_TFDMemTable: Create json file of TFDMemTable format from CSV file.

f:id:mojeld:20170901174729p:plain

Reduce development time and get to market faster with RAD Studio, Delphi, or C++Builder.
Design. Code. Compile. Deploy.

Start Free Trial Upgrade Today

Free Delphi Community Edition Free C++Builder Community Edition

Convert CSV file to TFDMemTable format JSON file.

Convert from CSV file to “JSON file” of “TFDMemTable format”.

include

code

Leave a ReplyCancel reply

Search

Something Fresh

Anuncio de la disponibilidad de RAD Studio 13 Florence

C++Builder 13 trae el lenguaje y las librerías ISO C++23 al compilador moderno de C++ para Win64.

Cómo usar RAD Studio con iOS 26 y XCode 26

Popular Posts

Hoja de ruta de RAD Studio noviembre de 2020

Categories

Popular From News

26 años ... de Delphi

Nuevo para el IDE en RAD Studio 10.4.1

RAD Studio con Delphi - ¡El código bajo original!

Tutorial de desarrollo de aplicaciones de Windows 10 para principiantes

Tkinter vs DelphiFMX en Embarcadero Open Source Stream

Latest From GetItNow

C++Builder @ stackoverflow

Delphi @ stackoverflow

InterBase @ stackoverflow

Categories

Useful Links

Follow us

Convert CSV file to TFDMemTable format JSON file.

Convert from CSV file to “JSON file” of “TFDMemTable format”.

include

code

Leave a ReplyCancel reply

Join Our Global Developer Community

Search

Something Fresh

Popular Posts

Categories

Popular From News

Categories

Useful Links

Follow us