Question

The following is in C++. The instructions for completion are at the bottom.

#include
#include
#include
#include
#include
#include

// function to determine if a token
vector is_token(const string& s) {
}

int main() {
// make this into a loop such that, like a terminal, input is accepted until the user hits CTRL+d
string str;
cout<<">";
getline(cin, str);

//vector to store the strings or characters returned by is_token() function
vector tokens;

tokens = is_token(str);
}

Modify the main() function to loop until the user exits (e.g. by pressing CTRL+d).

Write a function named is_token() that accepts a string of user input and determines what is a token (see below for token determination rules).
The function must return the separated tokens.

Determining if something is a token:

1. Whitespace (space, tab, etc) ends a token and is skipped unless it is in quotes (see below). You will find the function int isspace(char) useful for detecting whitespace characters.
2. Some characters are considered special tokens: | ; < > &. When one of these characters is encountered, the token being built is completed and a new token consisting of the character is created.
3. The character \ is a special character, i.e. an escape character. The next character will be part (or the start) of a token.
4. Items between single or double quotes are treated as parts of the current token.
5. Characters that do not meet the above rules are added on to the current token being built.

Some examples to test your output:

Input: > we\'ll do some "crazy \"air quotes\""
Output (tokens separated by commas): we'll, do, some, crazy "air quotes"

Input: > ls | sort >sorted_files.txt
Output (tokens separated by commas): ls, |, sort, >, sorted_files.txt

Input: > "now\;is"
Output: now;is

Answer 1

Solution:

1.Get the length of input string

2.in a loop, start validating each character of the input string

a.if the character encountered is space , token construction is completed.
      
a1.Remove the '' if any in the current token newly constructed
a2.Add the new token to vector
a3.if charater is encountered is one of |,;,<,>,& , treat them as 1 token and add it to the tokens vector
a4.if it is a normal character other than above, continue constructing the token
a5.if '' is encountered ,add the next character to the current token

3.last token, remove '' if any and add the token to vector
4.Parsing the input string into tokens(as per the requirement) is completed hence return the tokens vector.

output:

Source code:

// stringtokens.cpp : Defines the entry point for the console application.
//

#include<iostream>
#include<vector>
#include<string>
using namespace std;
// function to determine if a token
vector<string> is_token(const string& s)
{
   vector<string> tokens;

   //Get the length of input string

   int nLen = s.length();
  
   //variable to hold each token parsed
   string token;
  
   for(int i=0;i<nLen; i++)
   {
       //if the character encountered is space , token construction is completed.
      
       if (isspace(s[i]))
       {
           //Remove the '' if any
           size_t spos = token.find("\", 0);
           if (spos != string::npos)
           {
               token.replace(spos, 1, "");
           }

           //Add the new token to vector
           tokens.push_back(token);
           token.clear();
       }
       //if charater is encountered is one of |,;,<,>,& , treat them as 1 token and add it to the tokens vector
       else if((s[i] == '|') || (s[i] == ';') || (s[i] == '<') || (s[i] == '>') || (s[i] == '&'))              
       {
           tokens.push_back(token);
           token.clear();
           string temp;
           temp.push_back(s[i]);
           tokens.push_back(temp);
       }
       else //normal character other than above, continue constructing the token
       {
           token.push_back(s[i]);
       }
       //if '' is encountered ,add the next character to the current token
       if (s[i] == '\')
       {
           i++;
           token.push_back(s[i]);
       }
   }

   //last token, remove '' if any and add the token to vector
   size_t spos = token.find("\", 0);
   if (spos != string::npos)
   {
       token.replace(spos, 1, "");
   }

   tokens.push_back(token);

   //return the tokens vector
   return tokens;
}

int main() {
   // loop to get input is accepted until the user hits CTRL+d
   while (true)
   {
       string str;
       cout << "(Ctrl+D) to exit >";
       getline(cin, str);

       //if user pressed Ctrl+D exit out of loop
       if (str.compare("x4") == 0) break;

       //vector to store the strings or characters returned by is_token() function
       vector<string> tokens;

       tokens = is_token(str);

       cout << " Output tokens: ";
       for (int i = 0; i < tokens.size(); i++)
       {
           cout << tokens[i] << " ";
       }
       cout << endl;
   }
}