Code:
/ Net / Net / 3.5.50727.3053 / DEVDIV / depot / DevDiv / releases / Orcas / SP / wpf / src / Shared / MS / Internal / TokenizerHelper.cs / 1 / TokenizerHelper.cs
//----------------------------------------------------------------------------
//
// Copyright (C) Microsoft Corporation. All rights reserved.
//
// File: TokenizerHelper.cs
//
// Description: This file contains the implementation of TokenizerHelper.
// This class should be used by most - if not all - MIL parsers.
//
// History:
// 05/19/2003 : adsmith - Created it
// 05/20/2003 : adsmith - Moved to Shared
//
//---------------------------------------------------------------------------
using System;
using System.Collections;
using System.Diagnostics;
using System.ComponentModel;
using System.Globalization;
#if !PBTCOMPILER
using System.Windows;
#endif
#if WINDOWS_BASE
using MS.Internal.WindowsBase;
#elif PRESENTATION_CORE
using MS.Internal.PresentationCore;
#elif PRESENTATIONFRAMEWORK
using MS.Internal.PresentationFramework;
#elif PBTCOMPILER
using MS.Utility ;
using MS.Internal.Markup;
#elif DRT
using MS.Internal.Drt;
#else
#error Attempt to use FriendAccessAllowedAttribute from an unknown assembly.
using MS.Internal.YourAssemblyName;
#endif
#if PBTCOMPILER
namespace MS.Internal.Markup
#else
namespace MS.Internal
#endif
{
#if !PBTCOMPILER
[FriendAccessAllowed]
#endif
internal class TokenizerHelper
{
///
/// Constructor for TokenizerHelper which accepts an IFormatProvider.
/// If the IFormatProvider is null, we use the thread's IFormatProvider info.
/// We will use ',' as the list separator, unless it's the same as the
/// decimal separator. If it *is*, then we can't determine if, say, "23,5" is one
/// number or two. In this case, we will use ";" as the separator.
///
/// The string which will be tokenized.
/// The IFormatProvider which controls this tokenization.
internal TokenizerHelper(string str, IFormatProvider formatProvider)
{
char numberSeparator = GetNumericListSeparator(formatProvider);
Initialize(str, '\'', numberSeparator);
}
///
/// Initialize the TokenizerHelper with the string to tokenize,
/// the char which represents quotes and the list separator.
///
/// The string to tokenize.
/// The quote char.
/// The list separator.
internal TokenizerHelper(string str,
char quoteChar,
char separator)
{
Initialize(str, quoteChar, separator);
}
///
/// Initialize the TokenizerHelper with the string to tokenize,
/// the char which represents quotes and the list separator.
///
/// The string to tokenize.
/// The quote char.
/// The list separator.
private void Initialize(string str,
char quoteChar,
char separator)
{
_str = str;
_strLen = str == null ? 0 : str.Length;
_currentTokenIndex = -1;
_quoteChar = quoteChar;
_argSeparator = separator;
// immediately forward past any whitespace so
// NextToken() logic always starts on the first
// character of the next token.
while (_charIndex < _strLen)
{
if (!Char.IsWhiteSpace(_str, _charIndex))
{
break;
}
++_charIndex;
}
}
internal string GetCurrentToken()
{
// if no current token, return null
if (_currentTokenIndex < 0)
{
return null;
}
return _str.Substring(_currentTokenIndex,_currentTokenLength);
}
///
/// Throws an exception if there is any non-whitespace left un-parsed.
///
internal void LastTokenRequired()
{
if (_charIndex != _strLen)
{
throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperExtraDataEncountered));
}
}
///
/// Advances to the NextToken
///
/// true if next token was found, false if at end of string
internal bool NextToken()
{
return NextToken(false);
}
///
/// Advances to the NextToken, throwing an exception if not present
///
/// The next token found
internal string NextTokenRequired()
{
if (!NextToken(false))
{
throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperPrematureStringTermination));
}
return GetCurrentToken();
}
///
/// Advances to the NextToken, throwing an exception if not present
///
/// The next token found
internal string NextTokenRequired(bool allowQuotedToken)
{
if (!NextToken(allowQuotedToken))
{
throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperPrematureStringTermination));
}
return GetCurrentToken();
}
///
/// Advances to the NextToken
///
/// true if next token was found, false if at end of string
internal bool NextToken(bool allowQuotedToken)
{
// use the currently-set separator character.
return NextToken(allowQuotedToken, _argSeparator);
}
///
/// Advances to the NextToken. A separator character can be specified
/// which overrides the one previously set.
///
/// true if next token was found, false if at end of string
internal bool NextToken(bool allowQuotedToken, char separator)
{
_currentTokenIndex = -1; // reset the currentTokenIndex
_foundSeparator = false; // reset
// If we're at end of the string, just return false.
if (_charIndex >= _strLen)
{
return false;
}
char currentChar = _str[_charIndex];
Debug.Assert(!Char.IsWhiteSpace(currentChar),"Token started on Whitespace");
// setup the quoteCount
int quoteCount = 0;
// If we are allowing a quoted token and this token begins with a quote,
// set up the quote count and skip the initial quote
if (allowQuotedToken &&
currentChar == _quoteChar)
{
quoteCount++; // increment quote count
++_charIndex; // move to next character
}
int newTokenIndex = _charIndex;
int newTokenLength = 0;
// loop until hit end of string or hit a , or whitespace
// if at end of string ust return false.
while (_charIndex < _strLen)
{
currentChar = _str[_charIndex];
// if have a QuoteCount and this is a quote
// decrement the quoteCount
if (quoteCount > 0)
{
// if anything but a quoteChar we move on
if (currentChar == _quoteChar)
{
--quoteCount;
// if at zero which it always should for now
// break out of the loop
if (0 == quoteCount)
{
++_charIndex; // move past the quote
break;
}
}
}
else if ((Char.IsWhiteSpace(currentChar)) || (currentChar == separator))
{
if (currentChar == separator)
{
_foundSeparator = true;
}
break;
}
++_charIndex;
++newTokenLength;
}
// if quoteCount isn't zero we hit the end of the string
// before the ending quote
if (quoteCount > 0)
{
throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperMissingEndQuote));
}
ScanToNextToken(separator); // move so at the start of the nextToken for next call
// finally made it, update the _currentToken values
_currentTokenIndex = newTokenIndex;
_currentTokenLength = newTokenLength;
if (_currentTokenLength < 1)
{
throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperEmptyToken));
}
return true;
}
// helper to move the _charIndex to the next token or to the end of the string
void ScanToNextToken(char separator)
{
// if already at end of the string don't bother
if (_charIndex < _strLen)
{
char currentChar = _str[_charIndex];
// check that the currentChar is a space or the separator. If not
// we have an error. this can happen in the quote case
// that the char after the quotes string isn't a char.
if (!(currentChar == separator) &&
!Char.IsWhiteSpace(currentChar))
{
throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperExtraDataEncountered));
}
// loop until hit a character that isn't
// an argument separator or whitespace.
// !!!Todo: if more than one argSet throw an exception
int argSepCount = 0;
while (_charIndex < _strLen)
{
currentChar = _str[_charIndex];
if (currentChar == separator)
{
_foundSeparator = true;
++argSepCount;
_charIndex++;
if (argSepCount > 1)
{
throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperEmptyToken));
}
}
else if (Char.IsWhiteSpace(currentChar))
{
++_charIndex;
}
else
{
break;
}
}
// if there was a separatorChar then we shouldn't be
// at the end of string or means there was a separator
// but there isn't an arg
if (argSepCount > 0 && _charIndex >= _strLen)
{
throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperEmptyToken));
}
}
}
// Helper to get the numeric list separator for a given IFormatProvider.
// Separator is a comma [,] if the decimal separator is not a comma, or a semicolon [;] otherwise.
static internal char GetNumericListSeparator(IFormatProvider provider)
{
char numericSeparator = ',';
// Get the NumberFormatInfo out of the provider, if possible
// If the IFormatProvider doesn't not contain a NumberFormatInfo, then
// this method returns the current culture's NumberFormatInfo.
NumberFormatInfo numberFormat = NumberFormatInfo.GetInstance(provider);
Debug.Assert(null != numberFormat);
// Is the decimal separator is the same as the list separator?
// If so, we use the ";".
if ((numberFormat.NumberDecimalSeparator.Length > 0) && (numericSeparator == numberFormat.NumberDecimalSeparator[0]))
{
numericSeparator = ';';
}
return numericSeparator;
}
internal bool FoundSeparator
{
get
{
return _foundSeparator;
}
}
char _quoteChar;
char _argSeparator;
string _str;
int _strLen;
int _charIndex;
internal int _currentTokenIndex;
internal int _currentTokenLength;
bool _foundSeparator;
}
}
// File provided for Reference Use Only by Microsoft Corporation (c) 2007.
// Copyright (c) Microsoft Corporation. All rights reserved.
//----------------------------------------------------------------------------
//
// Copyright (C) Microsoft Corporation. All rights reserved.
//
// File: TokenizerHelper.cs
//
// Description: This file contains the implementation of TokenizerHelper.
// This class should be used by most - if not all - MIL parsers.
//
// History:
// 05/19/2003 : adsmith - Created it
// 05/20/2003 : adsmith - Moved to Shared
//
//---------------------------------------------------------------------------
using System;
using System.Collections;
using System.Diagnostics;
using System.ComponentModel;
using System.Globalization;
#if !PBTCOMPILER
using System.Windows;
#endif
#if WINDOWS_BASE
using MS.Internal.WindowsBase;
#elif PRESENTATION_CORE
using MS.Internal.PresentationCore;
#elif PRESENTATIONFRAMEWORK
using MS.Internal.PresentationFramework;
#elif PBTCOMPILER
using MS.Utility ;
using MS.Internal.Markup;
#elif DRT
using MS.Internal.Drt;
#else
#error Attempt to use FriendAccessAllowedAttribute from an unknown assembly.
using MS.Internal.YourAssemblyName;
#endif
#if PBTCOMPILER
namespace MS.Internal.Markup
#else
namespace MS.Internal
#endif
{
#if !PBTCOMPILER
[FriendAccessAllowed]
#endif
internal class TokenizerHelper
{
///
/// Constructor for TokenizerHelper which accepts an IFormatProvider.
/// If the IFormatProvider is null, we use the thread's IFormatProvider info.
/// We will use ',' as the list separator, unless it's the same as the
/// decimal separator. If it *is*, then we can't determine if, say, "23,5" is one
/// number or two. In this case, we will use ";" as the separator.
///
/// The string which will be tokenized.
/// The IFormatProvider which controls this tokenization.
internal TokenizerHelper(string str, IFormatProvider formatProvider)
{
char numberSeparator = GetNumericListSeparator(formatProvider);
Initialize(str, '\'', numberSeparator);
}
///
/// Initialize the TokenizerHelper with the string to tokenize,
/// the char which represents quotes and the list separator.
///
/// The string to tokenize.
/// The quote char.
/// The list separator.
internal TokenizerHelper(string str,
char quoteChar,
char separator)
{
Initialize(str, quoteChar, separator);
}
///
/// Initialize the TokenizerHelper with the string to tokenize,
/// the char which represents quotes and the list separator.
///
/// The string to tokenize.
/// The quote char.
/// The list separator.
private void Initialize(string str,
char quoteChar,
char separator)
{
_str = str;
_strLen = str == null ? 0 : str.Length;
_currentTokenIndex = -1;
_quoteChar = quoteChar;
_argSeparator = separator;
// immediately forward past any whitespace so
// NextToken() logic always starts on the first
// character of the next token.
while (_charIndex < _strLen)
{
if (!Char.IsWhiteSpace(_str, _charIndex))
{
break;
}
++_charIndex;
}
}
internal string GetCurrentToken()
{
// if no current token, return null
if (_currentTokenIndex < 0)
{
return null;
}
return _str.Substring(_currentTokenIndex,_currentTokenLength);
}
///
/// Throws an exception if there is any non-whitespace left un-parsed.
///
internal void LastTokenRequired()
{
if (_charIndex != _strLen)
{
throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperExtraDataEncountered));
}
}
///
/// Advances to the NextToken
///
/// true if next token was found, false if at end of string
internal bool NextToken()
{
return NextToken(false);
}
///
/// Advances to the NextToken, throwing an exception if not present
///
/// The next token found
internal string NextTokenRequired()
{
if (!NextToken(false))
{
throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperPrematureStringTermination));
}
return GetCurrentToken();
}
///
/// Advances to the NextToken, throwing an exception if not present
///
/// The next token found
internal string NextTokenRequired(bool allowQuotedToken)
{
if (!NextToken(allowQuotedToken))
{
throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperPrematureStringTermination));
}
return GetCurrentToken();
}
///
/// Advances to the NextToken
///
/// true if next token was found, false if at end of string
internal bool NextToken(bool allowQuotedToken)
{
// use the currently-set separator character.
return NextToken(allowQuotedToken, _argSeparator);
}
///
/// Advances to the NextToken. A separator character can be specified
/// which overrides the one previously set.
///
/// true if next token was found, false if at end of string
internal bool NextToken(bool allowQuotedToken, char separator)
{
_currentTokenIndex = -1; // reset the currentTokenIndex
_foundSeparator = false; // reset
// If we're at end of the string, just return false.
if (_charIndex >= _strLen)
{
return false;
}
char currentChar = _str[_charIndex];
Debug.Assert(!Char.IsWhiteSpace(currentChar),"Token started on Whitespace");
// setup the quoteCount
int quoteCount = 0;
// If we are allowing a quoted token and this token begins with a quote,
// set up the quote count and skip the initial quote
if (allowQuotedToken &&
currentChar == _quoteChar)
{
quoteCount++; // increment quote count
++_charIndex; // move to next character
}
int newTokenIndex = _charIndex;
int newTokenLength = 0;
// loop until hit end of string or hit a , or whitespace
// if at end of string ust return false.
while (_charIndex < _strLen)
{
currentChar = _str[_charIndex];
// if have a QuoteCount and this is a quote
// decrement the quoteCount
if (quoteCount > 0)
{
// if anything but a quoteChar we move on
if (currentChar == _quoteChar)
{
--quoteCount;
// if at zero which it always should for now
// break out of the loop
if (0 == quoteCount)
{
++_charIndex; // move past the quote
break;
}
}
}
else if ((Char.IsWhiteSpace(currentChar)) || (currentChar == separator))
{
if (currentChar == separator)
{
_foundSeparator = true;
}
break;
}
++_charIndex;
++newTokenLength;
}
// if quoteCount isn't zero we hit the end of the string
// before the ending quote
if (quoteCount > 0)
{
throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperMissingEndQuote));
}
ScanToNextToken(separator); // move so at the start of the nextToken for next call
// finally made it, update the _currentToken values
_currentTokenIndex = newTokenIndex;
_currentTokenLength = newTokenLength;
if (_currentTokenLength < 1)
{
throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperEmptyToken));
}
return true;
}
// helper to move the _charIndex to the next token or to the end of the string
void ScanToNextToken(char separator)
{
// if already at end of the string don't bother
if (_charIndex < _strLen)
{
char currentChar = _str[_charIndex];
// check that the currentChar is a space or the separator. If not
// we have an error. this can happen in the quote case
// that the char after the quotes string isn't a char.
if (!(currentChar == separator) &&
!Char.IsWhiteSpace(currentChar))
{
throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperExtraDataEncountered));
}
// loop until hit a character that isn't
// an argument separator or whitespace.
// !!!Todo: if more than one argSet throw an exception
int argSepCount = 0;
while (_charIndex < _strLen)
{
currentChar = _str[_charIndex];
if (currentChar == separator)
{
_foundSeparator = true;
++argSepCount;
_charIndex++;
if (argSepCount > 1)
{
throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperEmptyToken));
}
}
else if (Char.IsWhiteSpace(currentChar))
{
++_charIndex;
}
else
{
break;
}
}
// if there was a separatorChar then we shouldn't be
// at the end of string or means there was a separator
// but there isn't an arg
if (argSepCount > 0 && _charIndex >= _strLen)
{
throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperEmptyToken));
}
}
}
// Helper to get the numeric list separator for a given IFormatProvider.
// Separator is a comma [,] if the decimal separator is not a comma, or a semicolon [;] otherwise.
static internal char GetNumericListSeparator(IFormatProvider provider)
{
char numericSeparator = ',';
// Get the NumberFormatInfo out of the provider, if possible
// If the IFormatProvider doesn't not contain a NumberFormatInfo, then
// this method returns the current culture's NumberFormatInfo.
NumberFormatInfo numberFormat = NumberFormatInfo.GetInstance(provider);
Debug.Assert(null != numberFormat);
// Is the decimal separator is the same as the list separator?
// If so, we use the ";".
if ((numberFormat.NumberDecimalSeparator.Length > 0) && (numericSeparator == numberFormat.NumberDecimalSeparator[0]))
{
numericSeparator = ';';
}
return numericSeparator;
}
internal bool FoundSeparator
{
get
{
return _foundSeparator;
}
}
char _quoteChar;
char _argSeparator;
string _str;
int _strLen;
int _charIndex;
internal int _currentTokenIndex;
internal int _currentTokenLength;
bool _foundSeparator;
}
}
// File provided for Reference Use Only by Microsoft Corporation (c) 2007.
// Copyright (c) Microsoft Corporation. All rights reserved.
Link Menu
This book is available now!
Buy at Amazon US or
Buy at Amazon UK
- LambdaCompiler.ControlFlow.cs
- GeometryModel3D.cs
- CalculatedColumn.cs
- WindowsGraphicsCacheManager.cs
- LocalizableAttribute.cs
- BStrWrapper.cs
- IdentityNotMappedException.cs
- QilTernary.cs
- ResourcesBuildProvider.cs
- HttpCapabilitiesBase.cs
- BitmapMetadataEnumerator.cs
- HeaderCollection.cs
- Pair.cs
- XmlDataFileEditor.cs
- ToolboxItemFilterAttribute.cs
- ConfigurationManagerInternalFactory.cs
- KeyEvent.cs
- COAUTHIDENTITY.cs
- BitmapEffectCollection.cs
- OleDbWrapper.cs
- sqlmetadatafactory.cs
- SamlDelegatingWriter.cs
- EditorPartChrome.cs
- NameObjectCollectionBase.cs
- Decoder.cs
- SecurityAttributeGenerationHelper.cs
- HealthMonitoringSectionHelper.cs
- LinqDataSourceInsertEventArgs.cs
- SortedDictionary.cs
- BrowserTree.cs
- ObjectKeyFrameCollection.cs
- XmlCharacterData.cs
- XmlDataProvider.cs
- DoubleLinkListEnumerator.cs
- Stroke.cs
- VarRemapper.cs
- UserControl.cs
- PropertyTabAttribute.cs
- StoreConnection.cs
- TextRangeProviderWrapper.cs
- TdsParserSafeHandles.cs
- ToolStripButton.cs
- ApplyTemplatesAction.cs
- CoTaskMemHandle.cs
- FlowDocumentPaginator.cs
- ImageClickEventArgs.cs
- XmlIlVisitor.cs
- DrawingGroup.cs
- SplitterCancelEvent.cs
- OdbcConnectionHandle.cs
- formatter.cs
- ZipIOLocalFileHeader.cs
- PopupEventArgs.cs
- TextSelection.cs
- Encoder.cs
- ClaimSet.cs
- ByteConverter.cs
- COM2PropertyDescriptor.cs
- APCustomTypeDescriptor.cs
- TypeForwardedFromAttribute.cs
- Label.cs
- cookiecontainer.cs
- XmlDocumentFieldSchema.cs
- TextRunCache.cs
- altserialization.cs
- OperationAbortedException.cs
- TableLayoutRowStyleCollection.cs
- UnsafeNativeMethods.cs
- XmlSchemaAnyAttribute.cs
- Thread.cs
- ScriptingRoleServiceSection.cs
- StrokeNode.cs
- XmlSchemaInclude.cs
- ContractCodeDomInfo.cs
- Set.cs
- HashCodeCombiner.cs
- BindingCompleteEventArgs.cs
- COM2ExtendedBrowsingHandler.cs
- DragAssistanceManager.cs
- MLangCodePageEncoding.cs
- IdentifierElement.cs
- XmlRawWriter.cs
- UInt32Storage.cs
- BitStream.cs
- DataContext.cs
- ClientConfigurationHost.cs
- Int16Storage.cs
- XmlDocumentSerializer.cs
- InstalledFontCollection.cs
- HttpCacheParams.cs
- RefreshInfo.cs
- ObjectMemberMapping.cs
- RoutedEventArgs.cs
- ArrayConverter.cs
- PropertyMetadata.cs
- BlurBitmapEffect.cs
- CaseInsensitiveOrdinalStringComparer.cs
- Selection.cs
- CodeGenerator.cs
- CallTemplateAction.cs