Skip to content

Commit

Permalink
Change type to use ulong rather than ushort for the token index i…
Browse files Browse the repository at this point in the history 
…mplemented in HtmlData.cs.

This addresses issues where the `nextID` counter overflows ushort.MaxValue and wraps around to return the wrong index value, resulting in wrong strings output from the `Render` method.

Fix jamietre#204, jamietre#205, jamietre#189, jamietre#164
  • Loading branch information
@lukasbob
lukasbob committed May 3, 2016
1 parent 15c1cc3 commit cda8295
Show file tree
Hide file tree
Showing 31 changed files with 286 additions and 252 deletions.
2 changes: 1 addition & 1 deletion build/versions.bat
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
set csquery_version=1.3.5-beta5
set csquery_version=1.3.5-beta6
set csquery_mvc_version=1.3.11
set msbuild=C:\Windows\Microsoft.NET\Framework\v4.0.30319\msbuild
31 changes: 29 additions & 2 deletions source/CsQuery.Tests/Core/Index/SharedIndexTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ public void RunAllTests()
QueryByClass();
QueryByAttribute();
QueryByNodeName();
ManyTokensDoesNotCorruptIndex();
}


Expand Down Expand Up @@ -111,15 +112,41 @@ public void Remove()

}

public void ManyTokensDoesNotCorruptIndex()
{
// Parse and render multiple large documents with a lot of unique string tokens (in this case, class names).
// This ensures that the index is not corrupted at the boundary that it was previously at ~65535 - 256 unique strings,
// at which point the index counter would overflow and wrap around.
// Tests issue #204, #205, #189, #164.
CQ doc1 = Html(35000, 1);
CQ doc2 = Html(35000, 35001);
CQ doc3 = Html(35000, 70001);

Assert.That(doc1.Render(), Is.EqualTo(Html(35000, 1)));
Assert.That(doc2.Render(), Is.EqualTo(Html(35000, 35001))); // At this point the index counter would wrap and, due to the storage method in HtmlData, throw an OutOfRangeException.
Assert.That(doc3.Render(), Is.EqualTo(Html(35000, 70001))); // At this point the index would be corrupted, giving unpredictable results for new unique strings
}

private ushort[] GetKey(string what)
private ulong[] GetKey(string what)
{
var token = HtmlData.Tokenize(what.Substring(1));
ushort[] key = new ushort[2] { what[0], token };
var key = new [] { what[0], token };
return key;

}

private static string Html(int uniqueClassCount, int startAt)
{
var strb = new StringBuilder("<html><head></head><body>");
for (var i = 0; i < uniqueClassCount; i++)
{
strb.AppendFormat("<i class=\"{0}\"></i>", startAt + i);
}
strb.Append("</body></html>");
return strb.ToString();
}


private readonly string testHtml = @"
<html>
<div id=div1 data='container'>
Expand Down
2 changes: 1 addition & 1 deletion source/CsQuery/CQ_CsQuery/SetSelected.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ public CQ SetSelected(string groupName, IConvertible value)
}
if (item.Length > 0)
{
ushort nodeNameID = group[0].NodeNameID;
var nodeNameID = group[0].NodeNameID;
string type = group[0]["type"].ToUpper();
if (nodeNameID == HtmlData.tagOPTION)
{
Expand Down
2 changes: 1 addition & 1 deletion source/CsQuery/CsQuery.nuspec
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
<?xml version="1.0"?><package><metadata><id>CsQuery</id><version>1.3.5-beta5</version><title>CsQuery</title><authors>James Treworgy</authors><owners>James Treworgy</owners><licenseUrl>https://github.com/jamietre/CsQuery/blob/master/LICENSE.txt</licenseUrl><projectUrl>https://github.com/jamietre/CsQuery/</projectUrl><iconUrl>http://www.outsharked.com/csquery/images/csquery-icon-large.gif</iconUrl><requireLicenseAcceptance>false</requireLicenseAcceptance><summary>A complete CSS selector engine and jQuery port for .NET 4 and C#.</summary><description>CsQuery is an HTML parser, CSS selector engine and jQuery port for .NET 4 and C#. It implements all CSS2 and CSS3 selectors, all the DOM manipulation methods of jQuery, and some of the utility methods. </description><releaseNotes>
<?xml version="1.0"?><package><metadata><id>CsQuery</id><version>1.3.5-beta6</version><title>CsQuery</title><authors>James Treworgy</authors><owners>James Treworgy</owners><licenseUrl>https://github.com/jamietre/CsQuery/blob/master/LICENSE.txt</licenseUrl><projectUrl>https://github.com/jamietre/CsQuery/</projectUrl><iconUrl>http://www.outsharked.com/csquery/images/csquery-icon-large.gif</iconUrl><requireLicenseAcceptance>false</requireLicenseAcceptance><summary>A complete CSS selector engine and jQuery port for .NET 4 and C#.</summary><description>CsQuery is an HTML parser, CSS selector engine and jQuery port for .NET 4 and C#. It implements all CSS2 and CSS3 selectors, all the DOM manipulation methods of jQuery, and some of the utility methods. </description><releaseNotes>
Version 1.3.5 includes significant performance improvements and bug fixes.

Complete change log:
Expand Down
2 changes: 1 addition & 1 deletion source/CsQuery/Csquery.Signed.nuspec
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
<?xml version="1.0"?><package><metadata><id>CsQuery.Signed</id><version>1.3.5-beta5-signed</version><title>CsQuery.Signed</title><authors>James Treworgy</authors><owners>James Treworgy</owners><licenseUrl>https://github.com/jamietre/CsQuery/blob/master/LICENSE.txt</licenseUrl><projectUrl>https://github.com/jamietre/CsQuery/</projectUrl><iconUrl>http://www.outsharked.com/csquery/images/csquery-icon-large.gif</iconUrl><requireLicenseAcceptance>false</requireLicenseAcceptance><summary>A complete CSS selector engine and jQuery port for .NET 4 and C# (STRONG NAMED).</summary><description>CsQuery - Signed/Strong Named Edition. It's strongly advised that you use use the package 'CsQuery' unless you know why you want the signed edition.</description><releaseNotes>
<?xml version="1.0"?><package><metadata><id>CsQuery.Signed</id><version>1.3.5-beta6-signed</version><title>CsQuery.Signed</title><authors>James Treworgy</authors><owners>James Treworgy</owners><licenseUrl>https://github.com/jamietre/CsQuery/blob/master/LICENSE.txt</licenseUrl><projectUrl>https://github.com/jamietre/CsQuery/</projectUrl><iconUrl>http://www.outsharked.com/csquery/images/csquery-icon-large.gif</iconUrl><requireLicenseAcceptance>false</requireLicenseAcceptance><summary>A complete CSS selector engine and jQuery port for .NET 4 and C# (STRONG NAMED).</summary><description>CsQuery - Signed/Strong Named Edition. It's strongly advised that you use use the package 'CsQuery' unless you know why you want the signed edition.</description><releaseNotes>
Version 1.3.5 includes significant performance improvements and bug fixes.

Complete change log:
Expand Down
8 changes: 4 additions & 4 deletions source/CsQuery/Dom/IDomObject.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ public interface IDomObject : IDomNode, IComparable<IDomObject>
/// All the unique class names applied to this object.
/// </summary>
/// <value>
/// A sequence of strings
/// A sequence of strings
/// </value>

IEnumerable<string> Classes { get; }
Expand Down Expand Up @@ -736,13 +736,13 @@ public interface IDomObject : IDomNode, IComparable<IDomObject>
/// have to reindex when removing things)
/// </summary>

ushort NodePathID { get; }
ulong NodePathID { get; }

/// <summary>
/// Gets the full pathname of the node file.
/// </summary>

ushort[] NodePath { get; }
ulong[] NodePath { get; }

/// <summary>
/// Wrap this element in a CQ object. This is the CsQuery equivalent of the common jQuery
Expand Down Expand Up @@ -770,6 +770,6 @@ public interface IDomObject : IDomNode, IComparable<IDomObject>
/// The internal token ID for this element's node name.
/// </summary>

ushort NodeNameID { get; }
ulong NodeNameID { get; }
}
}
24 changes: 12 additions & 12 deletions source/CsQuery/Dom/Implementation/AttributeCollection.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,9 @@ public AttributeCollection()

#region private properties

private IDictionary<ushort, string> Attributes = new Dictionary<ushort, string>();
private IDictionary<ulong, string> Attributes = new Dictionary<ulong, string>();

internal string this[ushort nodeId]
internal string this[ulong nodeId]
{
get
{
Expand Down Expand Up @@ -151,7 +151,7 @@ public bool Remove(string name)
/// true if it succeeds, false if it fails.
/// </returns>

public bool Remove(ushort tokenId)
public bool Remove(ulong tokenId)
{
return Unset(tokenId);
}
Expand Down Expand Up @@ -209,7 +209,7 @@ public bool ContainsKey(string key)
/// true if it exists, false if not.
/// </returns>

public bool ContainsKey(ushort tokenId)
public bool ContainsKey(ulong tokenId)
{
return Attributes.ContainsKey(tokenId);
}
Expand Down Expand Up @@ -278,7 +278,7 @@ public bool TryGetValue(string name, out string value)
/// true if the key was present, false if not.
/// </returns>

public bool TryGetValue(ushort tokenId, out string value)
public bool TryGetValue(ulong tokenId, out string value)
{
// do not use trygetvalue from dictionary. We need default handling in Get
value = Get(tokenId);
Expand All @@ -296,7 +296,7 @@ public bool TryGetValue(ushort tokenId, out string value)

public void SetBoolean(string name)
{
ushort tokenId = HtmlData.Tokenize(name);
var tokenId = HtmlData.Tokenize(name);

SetBoolean(tokenId);
}
Expand All @@ -309,7 +309,7 @@ public void SetBoolean(string name)
/// The attribute's unique token ID
/// </param>

public void SetBoolean(ushort tokenId)
public void SetBoolean(ulong tokenId)
{
Attributes[tokenId] = null;
}
Expand Down Expand Up @@ -343,7 +343,7 @@ public bool Unset(string name)
/// true if it succeeds, false if it fails.
/// </returns>

public bool Unset(ushort tokenId)
public bool Unset(ulong tokenId)
{
bool result = Attributes.Remove(tokenId);
return result;
Expand All @@ -363,7 +363,7 @@ private string Get(string name)
return Get(HtmlData.Tokenize(name));
}

private string Get(ushort tokenId)
private string Get(ulong tokenId)
{
string value;

Expand Down Expand Up @@ -398,7 +398,7 @@ private void Set(string name, string value)
/// </summary>
/// <param name="tokenId"></param>
/// <param name="value"></param>
private void Set(ushort tokenId, string value)
private void Set(ulong tokenId, string value)
{
SetRaw(tokenId, value);
}
Expand All @@ -407,7 +407,7 @@ private void Set(ushort tokenId, string value)
/// </summary>
/// <param name="tokenId"></param>
/// <param name="value"></param>
internal void SetRaw(ushort tokenId, string value)
internal void SetRaw(ulong tokenId, string value)
{
if (value == null)
{
Expand All @@ -434,7 +434,7 @@ internal void SetRaw(ushort tokenId, string value)
yield return new KeyValuePair<string, string>(HtmlData.TokenName(kvp.Key).ToLower(), kvp.Value);
}
}
internal IEnumerable<ushort> GetAttributeIds()
internal IEnumerable<ulong> GetAttributeIds()
{
return Attributes.Keys;
}
Expand Down
14 changes: 7 additions & 7 deletions source/CsQuery/Dom/Implementation/CSSStyleDeclaration.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ public CSSStyleDeclaration(ICSSRule parentRule)

#region private properties

private IDictionary<ushort, string> _Styles;
private IDictionary<ulong, string> _Styles;
private string _QuickSetValue;

/// <summary>
Expand All @@ -95,13 +95,13 @@ public CSSStyleDeclaration(ICSSRule parentRule)
/// access is required.
/// </summary>

protected IDictionary<ushort, string> Styles
protected IDictionary<ulong, string> Styles
{
get
{
if (_Styles == null)
{
_Styles = new Dictionary<ushort, string>();
_Styles = new Dictionary<ulong, string>();
if (QuickSetValue != null)
{
AddStyles(QuickSetValue, false);
Expand Down Expand Up @@ -357,8 +357,8 @@ public CSSStyleDeclaration Clone()
}
else
{
IDictionary<ushort, string> styles = new Dictionary<ushort, string>();
foreach (KeyValuePair<ushort, string> kvp in Styles)
IDictionary<ulong, string> styles = new Dictionary<ulong, string>();
foreach (KeyValuePair<ulong, string> kvp in Styles)
{
styles.Add(kvp);
}
Expand Down Expand Up @@ -894,7 +894,7 @@ IEnumerator IEnumerable.GetEnumerator()
bool ICollection<KeyValuePair<string, string>>.Contains(KeyValuePair<string, string> item)
{
return HasStyleAttribute ?
Styles.Contains(new KeyValuePair<ushort, string>(HtmlData.Tokenize(item.Key), item.Value)) :
Styles.Contains(new KeyValuePair<ulong, string>(HtmlData.Tokenize(item.Key), item.Value)) :
false;
}

Expand All @@ -915,7 +915,7 @@ IEnumerator IEnumerable.GetEnumerator()
{
if (HasStyleAttribute)
{
var kvp = new KeyValuePair<ushort, string>(HtmlData.Tokenize(item.Key), item.Value);
var kvp = new KeyValuePair<ulong, string>(HtmlData.Tokenize(item.Key), item.Value);
return Styles.Remove(kvp);
}
else
Expand Down
4 changes: 2 additions & 2 deletions source/CsQuery/Dom/Implementation/DomDocument.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,11 +239,11 @@ internal set
/// The full path to this node. For Document nodes, this is always empty.
/// </summary>

public override ushort[] NodePath
public override ulong[] NodePath
{
get
{
return new ushort[] {};
return new ulong[] { };
}
}

Expand Down

0 comments on commit cda8295

Please sign in to comment.