diff -Nru libjavaewah-java-0.6.12/CHANGELOG libjavaewah-java-0.7.9/CHANGELOG --- libjavaewah-java-0.6.12/CHANGELOG 2013-06-11 17:48:37.000000000 +0000 +++ libjavaewah-java-0.7.9/CHANGELOG 2013-11-12 14:31:20.000000000 +0000 @@ -1,3 +1,43 @@ +version 0.7.9 (November 12th 2013) + - Spelled out the license in the source code to avoid any confusion + +version 0.7.8 (October 1st 2013) + - Resolved an issue with addStream... methods. + +version 0.7.7 (October 1st 2013) + - Resolved an issue with 32-bit EWAH caused by a partial update in version 0.7.6. + +version 0.7.6 (September 30th 2013) + - Fixed a memory leak + +version 0.7.5 (September 7th 2013) + - Fixed recently introduced "Get" method. Better unit testing. + +version 0.7.4 (September 7th 2013) + - Added a convenience method that allows you to query for the value of a bit (based on code by @zhenjl) + +version 0.7.3 (August 8th 2013) + - Fixed bug that could sometimes arise with the new iterator-based processing (O. Kaser) + + +version 0.7.2 (July 25th 2013) + - Clarifying the difference between setSizeInBits(int) and + setSizeInBits(int,boolean). Some code that worked previously + (but was potentially unsafe) might throw an exception when + using setSizeInBits(int). The fix is to call setSizeInBits(int,false) + which will be equivalent, but safer. + +version 0.7.1 (July 16th 2013) + - From now on, we require java 6 or better: in practice this means + that we are starting to use annotations such as @Override. + - We revert to the pre-0.7 behavior by setting the "sizeInBits" + of the bitmaps according to the max of the input bitmaps (issue 27). + + +version 0.7.0 (July 16th 2013) + - Performance improvements when aggregating several bitmaps + - Introduction of an iterator-based approach + version 0.6.12 (May 21st 2013) - Fixed bug with addStreamOfNegatedDirtyWords (Vicent Marti) - Removed some unnecessary recursion diff -Nru libjavaewah-java-0.6.12/debian/changelog libjavaewah-java-0.7.9/debian/changelog --- libjavaewah-java-0.6.12/debian/changelog 2017-06-30 04:38:56.000000000 +0000 +++ libjavaewah-java-0.7.9/debian/changelog 2019-02-28 22:09:54.000000000 +0000 @@ -1,3 +1,22 @@ +libjavaewah-java (0.7.9-1) unstable; urgency=medium + + * Team upload. + + [ Markus Koschany ] + * New upstream version 0.7.9. + * Use https for Format field. + * Use compat level 12. + * Declare compliance with Debian Policy 4.3.0. + * Use canonical VCS URI. + * Remove maven.cleanIgnoreRules and maven.publishedRules. + * Remove dh_install override. + * Remove debian/gbp.conf. xz compression is the default now. + + [ Emmanuel Bourg ] + * Removed the unused build dependency on libmaven-install-plugin-java + + -- Markus Koschany Thu, 28 Feb 2019 23:09:54 +0100 + libjavaewah-java (0.6.12-2) unstable; urgency=medium * Add missing build-dep on junit4 (Closes: #866546) diff -Nru libjavaewah-java-0.6.12/debian/compat libjavaewah-java-0.7.9/debian/compat --- libjavaewah-java-0.6.12/debian/compat 2017-06-30 04:38:56.000000000 +0000 +++ libjavaewah-java-0.7.9/debian/compat 2019-02-28 22:09:54.000000000 +0000 @@ -1 +1 @@ -10 +12 diff -Nru libjavaewah-java-0.6.12/debian/control libjavaewah-java-0.7.9/debian/control --- libjavaewah-java-0.6.12/debian/control 2017-06-30 04:38:56.000000000 +0000 +++ libjavaewah-java-0.7.9/debian/control 2019-02-28 22:09:54.000000000 +0000 @@ -2,19 +2,24 @@ Section: java Priority: optional Maintainer: Debian Java Maintainers -Uploaders: Jakub Adam , +Uploaders: + Jakub Adam , tony mancill -Build-Depends: debhelper (>= 10), default-jdk, maven-debian-helper (>= 1.6.3) -Build-Depends-Indep: libmaven-bundle-plugin-java, libmaven-install-plugin-java, - libmaven-javadoc-plugin-java, junit4 -Standards-Version: 4.0.0 +Build-Depends: + debhelper (>= 12), + default-jdk, + junit4, + libmaven-bundle-plugin-java, + libmaven-javadoc-plugin-java, + maven-debian-helper (>= 1.6.3) +Standards-Version: 4.3.0 +Vcs-Git: https://salsa.debian.org/java-team/libjavaewah-java.git +Vcs-Browser: https://salsa.debian.org/java-team/libjavaewah-java Homepage: https://github.com/lemire/javaewah -Vcs-Git: git://anonscm.debian.org/pkg-java/libjavaewah-java.git -Vcs-Browser: http://anonscm.debian.org/gitweb/?p=pkg-java/libjavaewah-java.git Package: libjavaewah-java Architecture: all -Depends: ${misc:Depends}, ${maven:Depends} +Depends: ${maven:Depends}, ${misc:Depends} Recommends: ${maven:OptionalDepends} Suggests: libjavaewah-java-doc Description: Compressed variant of the Java bitset class @@ -36,7 +41,7 @@ Package: libjavaewah-java-doc Architecture: all Section: doc -Depends: ${misc:Depends}, ${maven:DocDepends} +Depends: ${maven:DocDepends}, ${misc:Depends} Recommends: ${maven:DocOptionalDepends} Suggests: libjavaewah-java Description: Compressed variant of the Java bitset class (documentation) @@ -46,4 +51,3 @@ compresses better, but is not as fast. . This package contains the API documentation of libjavaewah-java. - diff -Nru libjavaewah-java-0.6.12/debian/copyright libjavaewah-java-0.7.9/debian/copyright --- libjavaewah-java-0.6.12/debian/copyright 2017-06-30 04:38:56.000000000 +0000 +++ libjavaewah-java-0.7.9/debian/copyright 2019-02-28 22:09:54.000000000 +0000 @@ -1,4 +1,4 @@ -Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: JavaEWAH Upstream-Contact: JavaEWAH team Source: https://github.com/lemire/javaewah diff -Nru libjavaewah-java-0.6.12/debian/gbp.conf libjavaewah-java-0.7.9/debian/gbp.conf --- libjavaewah-java-0.6.12/debian/gbp.conf 2017-06-30 04:38:56.000000000 +0000 +++ libjavaewah-java-0.7.9/debian/gbp.conf 1970-01-01 00:00:00.000000000 +0000 @@ -1,2 +0,0 @@ -[DEFAULT] -compression=xz diff -Nru libjavaewah-java-0.6.12/debian/maven.cleanIgnoreRules libjavaewah-java-0.7.9/debian/maven.cleanIgnoreRules --- libjavaewah-java-0.6.12/debian/maven.cleanIgnoreRules 2017-06-30 04:38:56.000000000 +0000 +++ libjavaewah-java-0.7.9/debian/maven.cleanIgnoreRules 1970-01-01 00:00:00.000000000 +0000 @@ -1 +0,0 @@ - diff -Nru libjavaewah-java-0.6.12/debian/maven.publishedRules libjavaewah-java-0.7.9/debian/maven.publishedRules --- libjavaewah-java-0.6.12/debian/maven.publishedRules 2017-06-30 04:38:56.000000000 +0000 +++ libjavaewah-java-0.7.9/debian/maven.publishedRules 1970-01-01 00:00:00.000000000 +0000 @@ -1 +0,0 @@ - diff -Nru libjavaewah-java-0.6.12/debian/rules libjavaewah-java-0.7.9/debian/rules --- libjavaewah-java-0.6.12/debian/rules 2017-06-30 04:38:56.000000000 +0000 +++ libjavaewah-java-0.7.9/debian/rules 2019-02-28 22:09:54.000000000 +0000 @@ -3,8 +3,3 @@ %: dh $@ --buildsystem=maven -# Can be removed once #711852 is fixed. -override_dh_install: - dh_install - # remove JARed javadoc, we already have unpacked version in /usr/share/doc - rm -rf debian/libjavaewah-java-doc/usr/share/maven-repo diff -Nru libjavaewah-java-0.6.12/example.java libjavaewah-java-0.7.9/example.java --- libjavaewah-java-0.6.12/example.java 2013-06-11 17:48:37.000000000 +0000 +++ libjavaewah-java-0.7.9/example.java 2013-11-12 14:31:20.000000000 +0000 @@ -1,9 +1,17 @@ import com.googlecode.javaewah.EWAHCompressedBitmap; import java.io.*; +/** + * @author lemire + * + */ public class example { - public static void main(String[] args) throws java.io.IOException { + /** + * @param args arguments from the command line + * @throws IOException if an IO error occurs + */ +public static void main(final String[] args) throws java.io.IOException { EWAHCompressedBitmap ewahBitmap1 = EWAHCompressedBitmap.bitmapOf(0,2,64,1<<30); EWAHCompressedBitmap ewahBitmap2 = EWAHCompressedBitmap.bitmapOf(1,3,64,1<<30); System.out.println("bitmap 1: "+ewahBitmap1); @@ -40,4 +48,5 @@ ewahBitmap1.readExternal(new ObjectInputStream(bis)); System.out.println("bitmap 1 (recovered) : "+ewahBitmap1); } + } diff -Nru libjavaewah-java-0.6.12/LICENSE libjavaewah-java-0.7.9/LICENSE --- libjavaewah-java-0.6.12/LICENSE 1970-01-01 00:00:00.000000000 +0000 +++ libjavaewah-java-0.7.9/LICENSE 2013-11-12 14:31:20.000000000 +0000 @@ -0,0 +1,191 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff -Nru libjavaewah-java-0.6.12/LICENSE-2.0.txt libjavaewah-java-0.7.9/LICENSE-2.0.txt --- libjavaewah-java-0.6.12/LICENSE-2.0.txt 1970-01-01 00:00:00.000000000 +0000 +++ libjavaewah-java-0.7.9/LICENSE-2.0.txt 2013-11-12 14:31:20.000000000 +0000 @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff -Nru libjavaewah-java-0.6.12/pom.xml libjavaewah-java-0.7.9/pom.xml --- libjavaewah-java-0.6.12/pom.xml 2013-06-11 17:48:37.000000000 +0000 +++ libjavaewah-java-0.7.9/pom.xml 2013-11-12 14:31:20.000000000 +0000 @@ -3,12 +3,12 @@ com.googlecode.javaewah JavaEWAH - 0.6.12 + 0.7.9 bundle - 1.5 - 1.5 + 1.6 + 1.6 UTF-8 diff -Nru libjavaewah-java-0.6.12/README libjavaewah-java-0.7.9/README --- libjavaewah-java-0.6.12/README 2013-06-11 17:48:37.000000000 +0000 +++ libjavaewah-java-0.7.9/README 1970-01-01 00:00:00.000000000 +0000 @@ -1,92 +0,0 @@ -JavaEWAH -(c) 2009-2013 -Daniel Lemire (http://lemire.me/en/), -Cliff Moon (https://github.com/cliffmoon), -David McIntosh (https://github.com/mctofu), -Robert Becho (https://github.com/RBecho), -Colby Ranger (https://github.com/crangeratgoogle) -and Veronika Zenz (https://github.com/veronikazenz) - - - -This code is licensed under Apache License, Version 2.0 (ASL2.0). -(GPL 2.0 derivatives are allowed.) - -This is a word-aligned compressed variant of -the Java Bitset class. We provide both a 64-bit -and a 32-bit RLE-like compression scheme. It can -be used to implement bitmap indexes. - -The goal of word-aligned compression is not to -achieve the best compression, but rather to -improve query processing time. Hence, we try -to save CPU cycles, maybe at the expense of -storage. However, the EWAH scheme we implemented -is always more efficient storage-wise than an -uncompressed bitmap (as implemented in the java -BitSet class by Sun). - - -For better performance, use a 64-bit JVM over -64-bit CPUs when using the 64-bit scheme (javaewah.EWAHCompressedBitmap). - -The 32-bit version (javaewah32.EWAHCompressedBitmap32) should -compress better but be comparatively slower. - - -For more details regarding the compression format, please -see Section 3 of the following paper: - -Daniel Lemire, Owen Kaser, Kamel Aouiche, Sorting improves word-aligned bitmap indexes. Data & Knowledge Engineering 69 (1), pages 3-28, 2010. - http://arxiv.org/abs/0901.3751 - - (The PDF file is freely available on the arXiv site.) - -== Benchmark == - -For a simple comparison between this library and other libraries such as -WAH, ConciseSet, BitSet and other options, please see - -https://github.com/lemire/simplebitmapbenchmark - -== Unit testing == - -As of October 2011, this packages relies on Maven. To -test it: - -mvn test - -See -http://maven.apache.org/guides/introduction/introduction-to-the-lifecycle.html -for details. - - -=== Usage == - -See example.java. - -== Maven central repository == - -You can download JavaEWAH from the Maven central repository: -http://repo1.maven.org/maven2/com/googlecode/javaewah/JavaEWAH/ - -You can also specify the dependency in the Maven "pom.xml" file: - - - - com.googlecode.javaewah - JavaEWAH - 0.6.11 - - - -Naturally, you should replace "version" by the version -you desire. - - - -== Clojure == - -Joel Boehland wrote Clojure wrappers: - -https://github.com/jolby/clojure-ewah-bitmap diff -Nru libjavaewah-java-0.6.12/README.md libjavaewah-java-0.7.9/README.md --- libjavaewah-java-0.6.12/README.md 1970-01-01 00:00:00.000000000 +0000 +++ libjavaewah-java-0.7.9/README.md 2013-11-12 14:31:20.000000000 +0000 @@ -0,0 +1,123 @@ +JavaEWAH +========================================================== + +(c) 2009-2013 +Daniel Lemire (http://lemire.me/en/), +Cliff Moon (https://github.com/cliffmoon), +David McIntosh (https://github.com/mctofu), +Robert Becho (https://github.com/RBecho), +Colby Ranger (https://github.com/crangeratgoogle) +Veronika Zenz (https://github.com/veronikazenz) +and Owen Kaser (https://github.com/owenkaser) + + +This code is licensed under Apache License, Version 2.0 (ASL2.0). +(GPL 2.0 derivatives are allowed.) + +This is a word-aligned compressed variant of +the Java Bitset class. We provide both a 64-bit +and a 32-bit RLE-like compression scheme. It can +be used to implement bitmap indexes. + +The goal of word-aligned compression is not to +achieve the best compression, but rather to +improve query processing time. Hence, we try +to save CPU cycles, maybe at the expense of +storage. However, the EWAH scheme we implemented +is always more efficient storage-wise than an +uncompressed bitmap (as implemented in the java +BitSet class by Sun). + + +For better performance, use a 64-bit JVM over +64-bit CPUs when using the 64-bit scheme (javaewah.EWAHCompressedBitmap). + +The 32-bit version (javaewah32.EWAHCompressedBitmap32) should +compress better but be comparatively slower. + +Java 6 or better is required. + +For more details regarding the compression format, please +see Section 3 of the following paper: + +Daniel Lemire, Owen Kaser, Kamel Aouiche, Sorting improves word-aligned bitmap indexes. Data & Knowledge Engineering 69 (1), pages 3-28, 2010. + http://arxiv.org/abs/0901.3751 + + (The PDF file is freely available on the arXiv site.) + +Benchmark +--------- + +For a simple comparison between this library and other libraries such as +WAH, ConciseSet, BitSet and other options, please see + +https://github.com/lemire/simplebitmapbenchmark + +Unit testing +------------ + +As of October 2011, this packages relies on Maven. To +test it: + +mvn test + +See +http://maven.apache.org/guides/introduction/introduction-to-the-lifecycle.html +for details. + + +Usage +----- + +See example.java. + +Maven central repository +------------------------ + +You can download JavaEWAH from the Maven central repository: +http://repo1.maven.org/maven2/com/googlecode/javaewah/JavaEWAH/ + +You can also specify the dependency in the Maven "pom.xml" file: + + + + com.googlecode.javaewah + JavaEWAH + 0.7.9 + + + +Naturally, you should replace "version" by the version +you desire. + +Travis (Continuous integration) +------------------------------- + +You can check whether the latest version builds on your favorite version +of Java using Travis: https://travis-ci.org/lemire/javaewah/builds/11059867 + +Clojure +------- + +Joel Boehland wrote Clojure wrappers: + +https://github.com/jolby/clojure-ewah-bitmap + +Frequent questions +------------------ + +Question: How do I check the value of a bit? + +Answer: If you need to routinely check the value of a given bit quickly, then +EWAH might not be the right format. However, if you must do it, you can proceed as +follows: + + /** + * Suppose you have the following bitmap: + */ + EWAHCompressedBitmap b = EWAHCompressedBitmap.bitmapOf(0,2,64,1<<30); + /** + * We want to know if bit 64 is set: + */ + boolean is64set = (b.and(EWAHCompressedBitmap.bitmapOf(64)).cardinality() == 1); + diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/benchmark/Benchmark32.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/benchmark/Benchmark32.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/benchmark/Benchmark32.java 1970-01-01 00:00:00.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/benchmark/Benchmark32.java 2013-11-12 14:31:20.000000000 +0000 @@ -0,0 +1,212 @@ +package com.googlecode.javaewah.benchmark; + +/* + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. + */ + +import java.text.DecimalFormat; +import java.util.List; +import com.googlecode.javaewah32.EWAHCompressedBitmap32; +import com.googlecode.javaewah.FastAggregation; +import com.googlecode.javaewah.IntIterator; +import com.googlecode.javaewah32.IteratingRLW32; +import com.googlecode.javaewah32.IteratorAggregation32; +import com.googlecode.javaewah32.IteratorUtil32; + +/** + * This class is used to benchmark the performance EWAH. + * + * @author Daniel Lemire + */ +public class Benchmark32 { + + @SuppressWarnings("javadoc") + public static void main(String args[]) { + test(100, 16, 1); +// test(2, 24, 1); + } + + @SuppressWarnings("javadoc") + public static void test(int N, int nbr, int repeat) { + DecimalFormat df = new DecimalFormat("0.###"); + ClusteredDataGenerator cdg = new ClusteredDataGenerator(); + for (int sparsity = 1; sparsity < 30 - nbr; sparsity += 2) { + long bogus = 0; + String line = ""; + long bef, aft; + line += sparsity; + int[][] data = new int[N][]; + int Max = (1 << (nbr + sparsity)); + System.out.println("# generating random data..."); + int[] inter = cdg.generateClustered(1 << (nbr/2), Max); + for (int k = 0; k < N; ++k) + data[k] = Benchmark.unite2by2(cdg.generateClustered(1 << nbr, Max),inter); + System.out.println("# generating random data... ok."); + // building + bef = System.currentTimeMillis(); + EWAHCompressedBitmap32[] ewah = new EWAHCompressedBitmap32[N]; + int size = 0; + for (int r = 0; r < repeat; ++r) { + size = 0; + for (int k = 0; k < N; ++k) { + ewah[k] = new EWAHCompressedBitmap32(); + for (int x = 0; x < data[k].length; ++x) { + ewah[k].set(data[k][x]); + } + size += ewah[k].sizeInBytes(); + } + } + aft = System.currentTimeMillis(); + line += "\t" + size; + line += "\t" + df.format((aft - bef) / 1000.0); + // uncompressing + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + int[] array = ewah[k].toArray(); + bogus += array.length; + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + // uncompressing + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + int[] array = new int[ewah[k].cardinality()]; + int c = 0; + for (int x : ewah[k]) + array[c++] = x; + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + // uncompressing + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + List L = ewah[k].getPositions(); + int[] array = new int[L.size()]; + int c = 0; + for (int x : L) + array[c++] = x; + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + // uncompressing + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + IntIterator iter = ewah[k].intIterator(); + while (iter.hasNext()) { + bogus += iter.next(); + } + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + line += "\t\t\t"; + // logical or + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + EWAHCompressedBitmap32 ewahor = ewah[0]; + for (int j = 1; j < k + 1; ++j) { + ewahor = ewahor.or(ewah[j]); + } + bogus += ewahor.sizeInBits(); + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + // fast logical or + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + EWAHCompressedBitmap32[] ewahcp = new EWAHCompressedBitmap32[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = ewah[j]; + } + EWAHCompressedBitmap32 ewahor = EWAHCompressedBitmap32 + .or(ewahcp); + bogus += ewahor.sizeInBits(); + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + + // fast logical or + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + EWAHCompressedBitmap32[] ewahcp = new EWAHCompressedBitmap32[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = ewah[j]; + } + EWAHCompressedBitmap32 ewahor = FastAggregation.or(ewahcp); + bogus += ewahor.sizeInBits(); + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + // fast logical or + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + IteratingRLW32[] ewahcp = new IteratingRLW32[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = ewah[j].getIteratingRLW(); + } + IteratingRLW32 ewahor = IteratorAggregation32.bufferedor(ewahcp); + bogus += IteratorUtil32.materialize(ewahor).sizeInBits(); + } + aft = System.currentTimeMillis(); + + line += "\t" + df.format((aft - bef) / 1000.0); + line += "\t\t\t"; + // logical and + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + EWAHCompressedBitmap32 ewahand = ewah[0]; + for (int j = 1; j < k + 1; ++j) { + ewahand = ewahand.and(ewah[j]); + } + bogus += ewahand.sizeInBits(); + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + // fast logical and + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + EWAHCompressedBitmap32[] ewahcp = new EWAHCompressedBitmap32[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = ewah[j]; + } + EWAHCompressedBitmap32 ewahand = EWAHCompressedBitmap32 + .and(ewahcp); + bogus += ewahand.sizeInBits(); + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + + + // fast logical and + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + IteratingRLW32[] ewahcp = new IteratingRLW32[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = ewah[j].getIteratingRLW(); + } + IteratingRLW32 ewahand = IteratorAggregation32.bufferedand(ewahcp); + bogus += IteratorUtil32.materialize(ewahand).sizeInBits(); + } + aft = System.currentTimeMillis(); + + line += "\t" + df.format((aft - bef) / 1000.0); + + + System.out + .println("time for building, toArray(), Java iterator, intIterator,\t\t\t logical or (2-by-2), logical or (grouped), FastAggregation.or, iterator-based or, \t\t\t (2-by-2) and, logical and (grouped), iterator-based and"); + System.out.println(line); + System.out.println("# bogus =" + bogus); + } + } +} diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/benchmark/BenchmarkIntersection32.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/benchmark/BenchmarkIntersection32.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/benchmark/BenchmarkIntersection32.java 1970-01-01 00:00:00.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/benchmark/BenchmarkIntersection32.java 2013-11-12 14:31:20.000000000 +0000 @@ -0,0 +1,130 @@ +package com.googlecode.javaewah.benchmark; + +import java.text.DecimalFormat; +import com.googlecode.javaewah32.*; + +/* + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. + */ +/** + * To benchmark the logical and (intersection) aggregate. + */ +public class BenchmarkIntersection32 { + + @SuppressWarnings("javadoc") + public static void main(String args[]) { + test(10, 18, 1); + } + + @SuppressWarnings({ "javadoc" }) + public static void test(int N, int nbr, int repeat) { + long bogus = 0; + + DecimalFormat df = new DecimalFormat("0.###"); + ClusteredDataGenerator cdg = new ClusteredDataGenerator(); + for (int sparsity = 1; sparsity < 30 - nbr; sparsity++) { + for (int times = 0; times < 2; ++times) { + String line = ""; + long bef, aft; + line += sparsity; + int[][] data = new int[N][]; + int Max = (1 << (nbr + sparsity)); + int[] inter = cdg.generateClustered(1 << (nbr/2), Max); + for (int k = 0; k < N; ++k) + data[k] = Benchmark.unite2by2(cdg.generateClustered(1 << nbr, Max),inter); + // building + EWAHCompressedBitmap32[] ewah = new EWAHCompressedBitmap32[N]; + for (int k = 0; k < N; ++k) { + ewah[k] = new EWAHCompressedBitmap32(); + for (int x = 0; x < data[k].length; ++x) { + ewah[k].set(data[k][x]); + } + data[k] = null; + } + // sanity check + if (true) { + EWAHCompressedBitmap32 answer = ewah[0].and(ewah[1]); + for (int k = 2; k < ewah.length; ++k) + answer = answer.and(ewah[k]); + + EWAHCompressedBitmap32 ewahand = EWAHCompressedBitmap32.and(ewah); + if (!answer.equals(ewahand)) + throw new RuntimeException( + "bug EWAHCompressedBitmap.and"); + EWAHCompressedBitmap32 ewahand2 = FastAggregation32 + .bufferedand(65536,ewah); + if (!ewahand.equals(ewahand2)) + throw new RuntimeException( + "bug FastAggregation.bufferedand "); + + } + + // logical or + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + EWAHCompressedBitmap32 ewahor = ewah[0]; + for (int j = 1; j < k + 1; ++j) { + ewahor = ewahor.and(ewah[j]); + } + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + + // fast logical or + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + EWAHCompressedBitmap32[] ewahcp = new EWAHCompressedBitmap32[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = ewah[j]; + } + EWAHCompressedBitmap32 ewahor = EWAHCompressedBitmap32 + .and(ewahcp); + bogus += ewahor.sizeInBits(); + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + // fast logical or + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + EWAHCompressedBitmap32[] ewahcp = new EWAHCompressedBitmap32[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = ewah[j]; + } + EWAHCompressedBitmap32 ewahor = FastAggregation32 + .bufferedand(65536,ewahcp); + bogus += ewahor.sizeInBits(); + } + aft = System.currentTimeMillis(); + + line += "\t" + df.format((aft - bef) / 1000.0); + // fast logical or + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + IteratingRLW32[] ewahcp = new IteratingRLW32[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = new IteratingBufferedRunningLengthWord32( + ewah[j]); + } + IteratingRLW32 ewahor = IteratorAggregation32.bufferedand(ewahcp); + int wordcounter = IteratorUtil32.cardinality(ewahor); + bogus += wordcounter; + } + aft = System.currentTimeMillis(); + + line += "\t" + df.format((aft - bef) / 1000.0); + + System.out + .println("# times for: 2by2 EWAHCompressedBitmap.and bufferedand iterator-bufferedand"); + + System.out.println(line); + } + System.out.println("# bogus =" + bogus); + + } + } +} diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/benchmark/BenchmarkIntersection.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/benchmark/BenchmarkIntersection.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/benchmark/BenchmarkIntersection.java 1970-01-01 00:00:00.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/benchmark/BenchmarkIntersection.java 2013-11-12 14:31:20.000000000 +0000 @@ -0,0 +1,130 @@ +package com.googlecode.javaewah.benchmark; + +import java.text.DecimalFormat; +import com.googlecode.javaewah.*; + +/* + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. + */ +/** + * To benchmark the logical and (intersection) aggregate. + */ +public class BenchmarkIntersection { + + @SuppressWarnings("javadoc") + public static void main(String args[]) { + test(10, 18, 1); + } + + @SuppressWarnings({ "javadoc"}) + public static void test(int N, int nbr, int repeat) { + long bogus = 0; + + DecimalFormat df = new DecimalFormat("0.###"); + ClusteredDataGenerator cdg = new ClusteredDataGenerator(); + for (int sparsity = 1; sparsity < 30 - nbr; sparsity++) { + for (int times = 0; times < 2; ++times) { + String line = ""; + long bef, aft; + line += sparsity; + int[][] data = new int[N][]; + int Max = (1 << (nbr + sparsity)); + int[] inter = cdg.generateClustered(1 << (nbr/2), Max); + for (int k = 0; k < N; ++k) + data[k] = Benchmark.unite2by2(cdg.generateClustered(1 << nbr, Max),inter); + // building + EWAHCompressedBitmap[] ewah = new EWAHCompressedBitmap[N]; + for (int k = 0; k < N; ++k) { + ewah[k] = new EWAHCompressedBitmap(); + for (int x = 0; x < data[k].length; ++x) { + ewah[k].set(data[k][x]); + } + data[k] = null; + } + // sanity check + if (true) { + EWAHCompressedBitmap answer = ewah[0].and(ewah[1]); + for (int k = 2; k < ewah.length; ++k) + answer = answer.and(ewah[k]); + + EWAHCompressedBitmap ewahand = EWAHCompressedBitmap.and(ewah); + if (!answer.equals(ewahand)) + throw new RuntimeException( + "bug EWAHCompressedBitmap.and"); + EWAHCompressedBitmap ewahand2 = FastAggregation + .bufferedand(65536,ewah); + if (!ewahand.equals(ewahand2)) + throw new RuntimeException( + "bug FastAggregation.bufferedand "); + + } + + // logical or + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + EWAHCompressedBitmap ewahor = ewah[0]; + for (int j = 1; j < k + 1; ++j) { + ewahor = ewahor.and(ewah[j]); + } + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + + // fast logical or + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + EWAHCompressedBitmap[] ewahcp = new EWAHCompressedBitmap[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = ewah[j]; + } + EWAHCompressedBitmap ewahor = EWAHCompressedBitmap + .and(ewahcp); + bogus += ewahor.sizeInBits(); + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + // fast logical or + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + EWAHCompressedBitmap[] ewahcp = new EWAHCompressedBitmap[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = ewah[j]; + } + EWAHCompressedBitmap ewahor = FastAggregation + .bufferedand(65536,ewahcp); + bogus += ewahor.sizeInBits(); + } + aft = System.currentTimeMillis(); + + line += "\t" + df.format((aft - bef) / 1000.0); + // fast logical or + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + IteratingRLW[] ewahcp = new IteratingRLW[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = new IteratingBufferedRunningLengthWord( + ewah[j]); + } + IteratingRLW ewahor = IteratorAggregation.bufferedand(ewahcp); + int wordcounter = IteratorUtil.cardinality(ewahor); + bogus += wordcounter; + } + aft = System.currentTimeMillis(); + + line += "\t" + df.format((aft - bef) / 1000.0); + + System.out + .println("# times for: 2by2 EWAHCompressedBitmap.and bufferedand iterator-bufferedand"); + + System.out.println(line); + } + System.out.println("# bogus =" + bogus); + + } + } +} diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/benchmark/Benchmark.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/benchmark/Benchmark.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/benchmark/Benchmark.java 2013-06-11 17:48:37.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/benchmark/Benchmark.java 2013-11-12 14:31:20.000000000 +0000 @@ -1,182 +1,284 @@ package com.googlecode.javaewah.benchmark; /* - * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc. and Veronika Zenz - * Licensed under APL 2.0. + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. */ - import java.text.DecimalFormat; +import java.util.Arrays; import java.util.List; import com.googlecode.javaewah.EWAHCompressedBitmap; import com.googlecode.javaewah.FastAggregation; import com.googlecode.javaewah.IntIterator; +import com.googlecode.javaewah.IteratingRLW; +import com.googlecode.javaewah.IteratorAggregation; +import com.googlecode.javaewah.IteratorUtil; /** - * This class is used to benchmark the performance - * EWAH. + * This class is used to benchmark the performance EWAH. * * @author Daniel Lemire */ public class Benchmark { + + /** + * Compute the union between two sorted arrays + * @param set1 first sorted array + * @param set2 second sorted array + * @return merged array + */ + static public int[] unite2by2(final int[] set1, final int[] set2) { + int pos = 0; + int k1 = 0, k2 = 0; + if (0 == set1.length) + return Arrays.copyOf(set2, set2.length); + if (0 == set2.length) + return Arrays.copyOf(set1, set1.length); + int[] buffer = new int[set1.length + set2.length]; + while (true) { + if (set1[k1] < set2[k2]) { + buffer[pos++] = set1[k1]; + ++k1; + if (k1 >= set1.length) { + for (; k2 < set2.length; ++k2) + buffer[pos++] = set2[k2]; + break; + } + } else if (set1[k1] == set2[k2]) { + buffer[pos++] = set1[k1]; + ++k1; + ++k2; + if (k1 >= set1.length) { + for (; k2 < set2.length; ++k2) + buffer[pos++] = set2[k2]; + break; + } + if (k2 >= set2.length) { + for (; k1 < set1.length; ++k1) + buffer[pos++] = set1[k1]; + break; + } + } else {// if (set1[k1]>set2[k2]) { + buffer[pos++] = set2[k2]; + ++k2; + if (k2 >= set2.length) { + for (; k1 < set1.length; ++k1) + buffer[pos++] = set1[k1]; + break; + } + } + } + return Arrays.copyOf(buffer, pos); + } + + + @SuppressWarnings("javadoc") + public static void main(String args[]) { + //test(2, 24, 1); + test(100, 16, 1); + } + + @SuppressWarnings("javadoc") + public static void test(int N, int nbr, int repeat) { + DecimalFormat df = new DecimalFormat("0.###"); + ClusteredDataGenerator cdg = new ClusteredDataGenerator(); + for (int sparsity = 1; sparsity < 30 - nbr; sparsity += 2) { + long bogus = 0; + String line = ""; + long bef, aft; + line += sparsity; + int[][] data = new int[N][]; + int Max = (1 << (nbr + sparsity)); + System.out.println("# generating random data..."); + int[] inter = cdg.generateClustered(1 << (nbr/2), Max); + for (int k = 0; k < N; ++k) + data[k] = unite2by2(cdg.generateClustered(1 << nbr, Max),inter); + System.out.println("# generating random data... ok."); + // building + bef = System.currentTimeMillis(); + EWAHCompressedBitmap[] ewah = new EWAHCompressedBitmap[N]; + int size = 0; + for (int r = 0; r < repeat; ++r) { + size = 0; + for (int k = 0; k < N; ++k) { + ewah[k] = new EWAHCompressedBitmap(); + for (int x = 0; x < data[k].length; ++x) { + ewah[k].set(data[k][x]); + } + size += ewah[k].sizeInBytes(); + } + } + aft = System.currentTimeMillis(); + line += "\t" + size; + line += "\t" + df.format((aft - bef) / 1000.0); + // uncompressing + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + int[] array = ewah[k].toArray(); + bogus += array.length; + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + // uncompressing + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + int[] array = new int[ewah[k].cardinality()]; + int c = 0; + for (int x : ewah[k]) + array[c++] = x; + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + // uncompressing + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + List L = ewah[k].getPositions(); + int[] array = new int[L.size()]; + int c = 0; + for (int x : L) + array[c++] = x; + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + // uncompressing + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + IntIterator iter = ewah[k].intIterator(); + while (iter.hasNext()) { + bogus += iter.next(); + } + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + line += "\t\t\t"; + // logical or + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + EWAHCompressedBitmap ewahor = ewah[0]; + for (int j = 1; j < k + 1; ++j) { + ewahor = ewahor.or(ewah[j]); + } + bogus += ewahor.sizeInBits(); + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + // fast logical or + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + EWAHCompressedBitmap[] ewahcp = new EWAHCompressedBitmap[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = ewah[j]; + } + EWAHCompressedBitmap ewahor = EWAHCompressedBitmap + .or(ewahcp); + bogus += ewahor.sizeInBits(); + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + + // fast logical or + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + EWAHCompressedBitmap[] ewahcp = new EWAHCompressedBitmap[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = ewah[j]; + } + EWAHCompressedBitmap ewahor = FastAggregation.or(ewahcp); + bogus += ewahor.sizeInBits(); + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + // fast logical or + // run sanity check + for (int k = 0; k < N; ++k) { + IteratingRLW[] ewahcp = new IteratingRLW[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = ewah[j].getIteratingRLW(); + } + IteratingRLW ewahor = IteratorAggregation.bufferedor(ewahcp); + EWAHCompressedBitmap ewahorp = EWAHCompressedBitmap.or(Arrays.copyOf(ewah, k+1)); + EWAHCompressedBitmap mewahor = IteratorUtil.materialize(ewahor); + if(!ewahorp.equals(mewahor)) throw new RuntimeException("bug"); + } + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + IteratingRLW[] ewahcp = new IteratingRLW[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = ewah[j].getIteratingRLW(); + } + IteratingRLW ewahor = IteratorAggregation.bufferedor(ewahcp); + bogus += IteratorUtil.materialize(ewahor).sizeInBits(); + } + aft = System.currentTimeMillis(); + + line += "\t" + df.format((aft - bef) / 1000.0); + line += "\t\t\t"; + // logical and + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + EWAHCompressedBitmap ewahand = ewah[0]; + for (int j = 1; j < k + 1; ++j) { + ewahand = ewahand.and(ewah[j]); + } + bogus += ewahand.sizeInBits(); + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + // fast logical and + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + EWAHCompressedBitmap[] ewahcp = new EWAHCompressedBitmap[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = ewah[j]; + } + EWAHCompressedBitmap ewahand = EWAHCompressedBitmap + .and(ewahcp); + bogus += ewahand.sizeInBits(); + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + + for (int k = 0; k < N; ++k) { + IteratingRLW[] ewahcp = new IteratingRLW[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = ewah[j].getIteratingRLW(); + } + IteratingRLW ewahand = IteratorAggregation.bufferedand(ewahcp); + EWAHCompressedBitmap ewahandp = EWAHCompressedBitmap.and(Arrays.copyOf(ewah, k+1)); + EWAHCompressedBitmap mewahand = IteratorUtil.materialize(ewahand); + if(!ewahandp.equals(mewahand)) throw new RuntimeException("bug"); + } + // fast logical and + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + IteratingRLW[] ewahcp = new IteratingRLW[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = ewah[j].getIteratingRLW(); + } + IteratingRLW ewahand = IteratorAggregation.bufferedand(ewahcp); + bogus += IteratorUtil.materialize(ewahand).sizeInBits(); + } + aft = System.currentTimeMillis(); + + line += "\t" + df.format((aft - bef) / 1000.0); - public static void main(String args[]) { - test(100, 16, 1); - } - - public static void test(int N, int nbr, int repeat) { - DecimalFormat df = new DecimalFormat("0.###"); - ClusteredDataGenerator cdg = new ClusteredDataGenerator(); - for (int sparsity = 1; sparsity < 30 - nbr; sparsity += 4) { - long bogus = 0; - String line = ""; - long bef, aft; - line += sparsity; - int[][] data = new int[N][]; - int Max = (1 << (nbr + sparsity)); - System.out.println("# generating random data..."); - for (int k = 0; k < N; ++k) - data[k] = cdg.generateClustered(1 << nbr, Max); - System.out.println("# generating random data... ok."); - // building - bef = System.currentTimeMillis(); - EWAHCompressedBitmap[] ewah = new EWAHCompressedBitmap[N]; - int size = 0; - for (int r = 0; r < repeat; ++r) { - size = 0; - for (int k = 0; k < N; ++k) { - ewah[k] = new EWAHCompressedBitmap(); - for (int x = 0; x < data[k].length; ++x) { - ewah[k].set(data[k][x]); - } - size += ewah[k].sizeInBytes(); - } - } - aft = System.currentTimeMillis(); - line += "\t" + size; - line += "\t" + df.format((aft - bef) / 1000.0); - // uncompressing - bef = System.currentTimeMillis(); - for (int r = 0; r < repeat; ++r) - for (int k = 0; k < N; ++k) { - int[] array = ewah[k].toArray(); - bogus += array.length; - } - aft = System.currentTimeMillis(); - line += "\t" + df.format((aft - bef) / 1000.0); - // uncompressing - bef = System.currentTimeMillis(); - for (int r = 0; r < repeat; ++r) - for (int k = 0; k < N; ++k) { - int[] array = new int[ewah[k].cardinality()]; - int c = 0; - for (int x : ewah[k]) - array[c++] = x; - } - aft = System.currentTimeMillis(); - line += "\t" + df.format((aft - bef) / 1000.0); - // uncompressing - bef = System.currentTimeMillis(); - for (int r = 0; r < repeat; ++r) - for (int k = 0; k < N; ++k) { - List L = ewah[k].getPositions(); - int[] array = new int[L.size()]; - int c = 0; - for (int x : L) - array[c++] = x; - } - aft = System.currentTimeMillis(); - line += "\t" + df.format((aft - bef) / 1000.0); - // uncompressing - bef = System.currentTimeMillis(); - for (int r = 0; r < repeat; ++r) - for (int k = 0; k < N; ++k) { - IntIterator iter = ewah[k].intIterator(); - while (iter.hasNext()) { - bogus += iter.next(); - } - } - aft = System.currentTimeMillis(); - line += "\t" + df.format((aft - bef) / 1000.0); - // logical or - bef = System.currentTimeMillis(); - for (int r = 0; r < repeat; ++r) - for (int k = 0; k < N; ++k) { - EWAHCompressedBitmap ewahor = ewah[0]; - for (int j = 1; j < k; ++j) { - ewahor = ewahor.or(ewah[j]); - } - } - aft = System.currentTimeMillis(); - line += "\t" + df.format((aft - bef) / 1000.0); - // fast logical or - bef = System.currentTimeMillis(); - for (int r = 0; r < repeat; ++r) - for (int k = 0; k < N; ++k) { - EWAHCompressedBitmap[] ewahcp = new EWAHCompressedBitmap[k + 1]; - for (int j = 0; j < k + 1; ++j) { - ewahcp[j] = ewah[j]; - } - EWAHCompressedBitmap ewahor = EWAHCompressedBitmap.or(ewahcp); - bogus += ewahor.sizeInBits(); - } - aft = System.currentTimeMillis(); - line += "\t" + df.format((aft - bef) / 1000.0); - - // fast logical or - bef = System.currentTimeMillis(); - for (int r = 0; r < repeat; ++r) - for (int k = 0; k < N; ++k) { - EWAHCompressedBitmap[] ewahcp = new EWAHCompressedBitmap[k + 1]; - for (int j = 0; j < k + 1; ++j) { - ewahcp[j] = ewah[j]; - } - EWAHCompressedBitmap ewahor = FastAggregation.or(ewahcp); - bogus += ewahor.sizeInBits(); - } - aft = System.currentTimeMillis(); - line += "\t" + df.format((aft - bef) / 1000.0); - - // logical and - bef = System.currentTimeMillis(); - for (int r = 0; r < repeat; ++r) - for (int k = 0; k < N; ++k) { - EWAHCompressedBitmap ewahand = ewah[0]; - for (int j = 1; j < k; ++j) { - ewahand = ewahand.and(ewah[j]); - } - } - aft = System.currentTimeMillis(); - line += "\t" + df.format((aft - bef) / 1000.0); - // fast logical and - bef = System.currentTimeMillis(); - for (int r = 0; r < repeat; ++r) - for (int k = 0; k < N; ++k) { - EWAHCompressedBitmap[] ewahcp = new EWAHCompressedBitmap[k + 1]; - for (int j = 0; j < k + 1; ++j) { - ewahcp[j] = ewah[j]; - } - EWAHCompressedBitmap ewahand = EWAHCompressedBitmap.and(ewahcp); - bogus += ewahand.sizeInBits(); - } - aft = System.currentTimeMillis(); - line += "\t" + df.format((aft - bef) / 1000.0); - // fast logical and - bef = System.currentTimeMillis(); - for (int r = 0; r < repeat; ++r) - for (int k = 0; k < N; ++k) { - EWAHCompressedBitmap[] ewahcp = new EWAHCompressedBitmap[k + 1]; - for (int j = 0; j < k + 1; ++j) { - ewahcp[j] = ewah[j]; - } - EWAHCompressedBitmap ewahand = FastAggregation.and(ewahcp); - bogus += ewahand.sizeInBits(); - } - aft = System.currentTimeMillis(); - line += "\t" + df.format((aft - bef) / 1000.0); - System.out.println("time for building, toArray(), Java iterator, intIterator, logical or (2-by-2), logical or (grouped), FastAggregation.or, and (2-by-2), logical and (grouped),FastAggregation.and"); - System.out.println(line); - System.out.println("# bogus =" + bogus); - } - } + + System.out + .println("time for building, toArray(), Java iterator, intIterator,\t\t\t logical or (2-by-2), logical or (grouped), FastAggregation.or, iterator-based or, \t\t\t (2-by-2) and, logical and (grouped), iterator-based and"); + System.out.println(line); + System.out.println("# bogus =" + bogus); + } + } } diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/benchmark/BenchmarkUnion32.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/benchmark/BenchmarkUnion32.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/benchmark/BenchmarkUnion32.java 1970-01-01 00:00:00.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/benchmark/BenchmarkUnion32.java 2013-11-12 14:31:20.000000000 +0000 @@ -0,0 +1,165 @@ +package com.googlecode.javaewah.benchmark; + +import java.text.DecimalFormat; + +import com.googlecode.javaewah.FastAggregation; +import com.googlecode.javaewah32.*; + +/* + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. + */ +/** + * To benchmark the logical or (union) aggregate. + */ +public class BenchmarkUnion32 { + + @SuppressWarnings("javadoc") + public static void main(String args[]) { + test(10, 18, 1); + } + + @SuppressWarnings({ "javadoc", "deprecation" }) + public static void test(int N, int nbr, int repeat) { + long bogus = 0; + + DecimalFormat df = new DecimalFormat("0.###"); + ClusteredDataGenerator cdg = new ClusteredDataGenerator(); + for (int sparsity = 1; sparsity < 30 - nbr; sparsity++) { + for (int times = 0; times < 2; ++times) { + String line = ""; + long bef, aft; + line += sparsity; + int[][] data = new int[N][]; + int Max = (1 << (nbr + sparsity)); + for (int k = 0; k < N; ++k) + data[k] = cdg.generateClustered(1 << nbr, Max); + // building + EWAHCompressedBitmap32[] ewah = new EWAHCompressedBitmap32[N]; + for (int k = 0; k < N; ++k) { + ewah[k] = new EWAHCompressedBitmap32(); + for (int x = 0; x < data[k].length; ++x) { + ewah[k].set(data[k][x]); + } + data[k] = null; + } + // sanity check + if(true){ + EWAHCompressedBitmap32 answer = ewah[0].or(ewah[1]); + for(int k = 2; k < ewah.length; ++k) + answer = answer.or(ewah[k]); + + EWAHCompressedBitmap32 ewahor = EWAHCompressedBitmap32 + .or(ewah); + if(!answer.equals(ewahor)) throw new RuntimeException("bug EWAHCompressedBitmap.or"); + EWAHCompressedBitmap32 ewahor3 = FastAggregation + .or(ewah); + if(!ewahor.equals(ewahor3)) throw new RuntimeException("bug FastAggregation.or"); + EWAHCompressedBitmap32 ewahor2 = FastAggregation32 + .bufferedor(65536,ewah); + if(!ewahor.equals(ewahor2)) throw new RuntimeException("bug FastAggregation.bufferedor "); + + } + + // logical or + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + EWAHCompressedBitmap32 ewahor = ewah[0]; + for (int j = 1; j < k + 1; ++j) { + ewahor = ewahor.or(ewah[j]); + } + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + + + // fast logical or + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + EWAHCompressedBitmap32[] ewahcp = new EWAHCompressedBitmap32[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = ewah[j]; + } + EWAHCompressedBitmap32 ewahor = EWAHCompressedBitmap32 + .or(ewahcp); + bogus += ewahor.sizeInBits(); + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + + // fast logical or + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + EWAHCompressedBitmap32[] ewahcp = new EWAHCompressedBitmap32[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = ewah[j]; + } + EWAHCompressedBitmap32 ewahor = FastAggregation + .or(ewahcp); + bogus += ewahor.sizeInBits(); + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + + + // fast logical or + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + EWAHCompressedBitmap32[] ewahcp = new EWAHCompressedBitmap32[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = ewah[j]; + } + EWAHCompressedBitmap32 ewahor = FastAggregation32 + .bufferedor(65536,ewahcp); + bogus += ewahor.sizeInBits(); + } + aft = System.currentTimeMillis(); + + line += "\t" + df.format((aft - bef) / 1000.0); + // fast logical or + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + EWAHCompressedBitmap32[] ewahcp = new EWAHCompressedBitmap32[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = ewah[j]; + } + EWAHCompressedBitmap32 x = new EWAHCompressedBitmap32(); + FastAggregation32.legacy_orWithContainer(x, ewahcp); + bogus += x.sizeInBits(); + } + aft = System.currentTimeMillis(); + + line += "\t" + df.format((aft - bef) / 1000.0); + // fast logical or + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + IteratingRLW32[] ewahcp = new IteratingRLW32[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = new IteratingBufferedRunningLengthWord32(ewah[j]); + } + IteratingRLW32 ewahor = IteratorAggregation32 + .bufferedor(ewahcp); + int wordcounter = IteratorUtil32.cardinality(ewahor); + bogus += wordcounter; + } + aft = System.currentTimeMillis(); + + line += "\t" + df.format((aft - bef) / 1000.0); + + + System.out + .println("# times for: 2by2 EWAHCompressedBitmap.or FastAggregation.or experimentalor bufferedor legacygroupedor iterator-bufferedor"); + + System.out.println(line); + } + System.out.println("# bogus =" + bogus); + + } + } +} diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/benchmark/BenchmarkUnion.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/benchmark/BenchmarkUnion.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/benchmark/BenchmarkUnion.java 1970-01-01 00:00:00.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/benchmark/BenchmarkUnion.java 2013-11-12 14:31:20.000000000 +0000 @@ -0,0 +1,164 @@ +package com.googlecode.javaewah.benchmark; + +import java.text.DecimalFormat; +import com.googlecode.javaewah.*; + +/* + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. + */ +/** + * To benchmark the logical or (union) aggregate. + */ +public class BenchmarkUnion { + + @SuppressWarnings("javadoc") + public static void main(String args[]) { + test(10, 18, 1); + } + + @SuppressWarnings({ "javadoc", "deprecation" }) + public static void test(int N, int nbr, int repeat) { + long bogus = 0; + + DecimalFormat df = new DecimalFormat("0.###"); + ClusteredDataGenerator cdg = new ClusteredDataGenerator(); + for (int sparsity = 1; sparsity < 30 - nbr; sparsity++) { + for (int times = 0; times < 2; ++times) { + String line = ""; + long bef, aft; + line += sparsity; + int[][] data = new int[N][]; + int Max = (1 << (nbr + sparsity)); + for (int k = 0; k < N; ++k) + data[k] = cdg.generateClustered(1 << nbr, Max); + // building + EWAHCompressedBitmap[] ewah = new EWAHCompressedBitmap[N]; + for (int k = 0; k < N; ++k) { + ewah[k] = new EWAHCompressedBitmap(); + for (int x = 0; x < data[k].length; ++x) { + ewah[k].set(data[k][x]); + } + data[k] = null; + } + // sanity check + if (true) { + EWAHCompressedBitmap answer = ewah[0].or(ewah[1]); + for (int k = 2; k < ewah.length; ++k) + answer = answer.or(ewah[k]); + + EWAHCompressedBitmap ewahor = EWAHCompressedBitmap.or(ewah); + if (!answer.equals(ewahor)) + throw new RuntimeException( + "bug EWAHCompressedBitmap.or"); + EWAHCompressedBitmap ewahor3 = FastAggregation.or(ewah); + if (!ewahor.equals(ewahor3)) + throw new RuntimeException("bug FastAggregation.or"); + EWAHCompressedBitmap ewahor2 = FastAggregation + .bufferedor(65536,ewah); + if (!ewahor.equals(ewahor2)) + throw new RuntimeException( + "bug FastAggregation.bufferedor "); + + } + + // logical or + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + EWAHCompressedBitmap ewahor = ewah[0]; + for (int j = 1; j < k + 1; ++j) { + ewahor = ewahor.or(ewah[j]); + } + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + + // fast logical or + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + EWAHCompressedBitmap[] ewahcp = new EWAHCompressedBitmap[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = ewah[j]; + } + EWAHCompressedBitmap ewahor = EWAHCompressedBitmap + .or(ewahcp); + bogus += ewahor.sizeInBits(); + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + + // fast logical or + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + EWAHCompressedBitmap[] ewahcp = new EWAHCompressedBitmap[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = ewah[j]; + } + EWAHCompressedBitmap ewahor = FastAggregation + .or(ewahcp); + bogus += ewahor.sizeInBits(); + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + + + // fast logical or + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + EWAHCompressedBitmap[] ewahcp = new EWAHCompressedBitmap[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = ewah[j]; + } + EWAHCompressedBitmap ewahor = FastAggregation + .bufferedor(65536,ewahcp); + bogus += ewahor.sizeInBits(); + } + aft = System.currentTimeMillis(); + + line += "\t" + df.format((aft - bef) / 1000.0); + // fast logical or + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + EWAHCompressedBitmap[] ewahcp = new EWAHCompressedBitmap[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = ewah[j]; + } + EWAHCompressedBitmap x = new EWAHCompressedBitmap(); + FastAggregation.legacy_orWithContainer(x, ewahcp); + bogus += x.sizeInBits(); + } + aft = System.currentTimeMillis(); + + line += "\t" + df.format((aft - bef) / 1000.0); + // fast logical or + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + IteratingRLW[] ewahcp = new IteratingRLW[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = new IteratingBufferedRunningLengthWord( + ewah[j]); + } + IteratingRLW ewahor = IteratorAggregation.bufferedor(ewahcp); + int wordcounter = IteratorUtil.cardinality(ewahor); + bogus += wordcounter; + } + aft = System.currentTimeMillis(); + + line += "\t" + df.format((aft - bef) / 1000.0); + + System.out + .println("# times for: 2by2 EWAHCompressedBitmap.or FastAggregation.or experimentalor bufferedor legacygroupedor iterator-bufferedor"); + + System.out.println(line); + } + System.out.println("# bogus =" + bogus); + + } + } +} diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/benchmark/BenchmarkXOR32.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/benchmark/BenchmarkXOR32.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/benchmark/BenchmarkXOR32.java 1970-01-01 00:00:00.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/benchmark/BenchmarkXOR32.java 2013-11-12 14:31:20.000000000 +0000 @@ -0,0 +1,137 @@ +package com.googlecode.javaewah.benchmark; + +import java.text.DecimalFormat; + +import com.googlecode.javaewah.FastAggregation; +import com.googlecode.javaewah32.*; + +/* + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. + */ +/** + * To benchmark the logical xor aggregate. + */ +public class BenchmarkXOR32 { + + @SuppressWarnings("javadoc") + public static void main(String args[]) { + test(10, 18, 1); + //test(2, 22, 1); + } + + @SuppressWarnings({ "javadoc" }) + public static void test(int N, int nbr, int repeat) { + long bogus = 0; + + DecimalFormat df = new DecimalFormat("0.###"); + ClusteredDataGenerator cdg = new ClusteredDataGenerator(); + for (int sparsity = 1; sparsity < 30 - nbr; sparsity++) { + for (int times = 0; times < 2; ++times) { + String line = ""; + long bef, aft; + line += sparsity; + int[][] data = new int[N][]; + int Max = (1 << (nbr + sparsity)); + for (int k = 0; k < N; ++k) + data[k] = cdg.generateClustered(1 << nbr, Max); + // building + EWAHCompressedBitmap32[] ewah = new EWAHCompressedBitmap32[N]; + for (int k = 0; k < N; ++k) { + ewah[k] = new EWAHCompressedBitmap32(); + for (int x = 0; x < data[k].length; ++x) { + ewah[k].set(data[k][x]); + } + data[k] = null; + } + // sanity check + if (true) { + EWAHCompressedBitmap32 answer = ewah[0].xor(ewah[1]); + for (int k = 2; k < ewah.length; ++k) + answer = answer.xor(ewah[k]); + EWAHCompressedBitmap32 ewahor3 = FastAggregation.xor(ewah); + if (!answer.equals(ewahor3)) + throw new RuntimeException("bug FastAggregation.xor"); + EWAHCompressedBitmap32 ewahor2 = FastAggregation32 + .bufferedxor(65536,ewah); + if (!answer.equals(ewahor2)) + throw new RuntimeException( + "bug FastAggregation.bufferedxor "); + EWAHCompressedBitmap32 iwah = IteratorUtil32.materialize(IteratorAggregation32.bufferedxor(IteratorUtil32.toIterators(ewah))); + if (!answer.equals(iwah)) + throw new RuntimeException( + "bug xor it "); + + } + + // logical xor + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + EWAHCompressedBitmap32 ewahor = ewah[0]; + for (int j = 1; j < k + 1; ++j) { + ewahor = ewahor.xor(ewah[j]); + } + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + + // fast logical xor + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + EWAHCompressedBitmap32[] ewahcp = new EWAHCompressedBitmap32[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = ewah[j]; + } + EWAHCompressedBitmap32 ewahor = FastAggregation + .xor(ewahcp); + bogus += ewahor.sizeInBits(); + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + + + // fast logical xor + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + EWAHCompressedBitmap32[] ewahcp = new EWAHCompressedBitmap32[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = ewah[j]; + } + EWAHCompressedBitmap32 ewahor = FastAggregation32 + .bufferedxor(65536,ewahcp); + bogus += ewahor.sizeInBits(); + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + + // fast logical xor + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + IteratingRLW32[] ewahcp = new IteratingRLW32[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = new IteratingBufferedRunningLengthWord32( + ewah[j]); + } + IteratingRLW32 ewahor = IteratorAggregation32.bufferedxor(ewahcp); + int wordcounter = IteratorUtil32.cardinality(ewahor); + bogus += wordcounter; + } + aft = System.currentTimeMillis(); + + line += "\t" + df.format((aft - bef) / 1000.0); + + + System.out + .println("# times for: 2by2 FastAggregation.xor bufferedxor iterator-based"); + + System.out.println(line); + } + System.out.println("# bogus =" + bogus); + + } + } +} diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/benchmark/BenchmarkXOR.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/benchmark/BenchmarkXOR.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/benchmark/BenchmarkXOR.java 1970-01-01 00:00:00.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/benchmark/BenchmarkXOR.java 2013-11-12 14:31:20.000000000 +0000 @@ -0,0 +1,134 @@ +package com.googlecode.javaewah.benchmark; + +import java.text.DecimalFormat; +import com.googlecode.javaewah.*; + +/* + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. + */ +/** + * To benchmark the logical xor aggregate. + */ +public class BenchmarkXOR { + + @SuppressWarnings("javadoc") + public static void main(String args[]) { + //test(10, 18, 1); + test(2, 22, 1); + } + + @SuppressWarnings({ "javadoc" }) + public static void test(int N, int nbr, int repeat) { + long bogus = 0; + + DecimalFormat df = new DecimalFormat("0.###"); + ClusteredDataGenerator cdg = new ClusteredDataGenerator(); + for (int sparsity = 1; sparsity < 30 - nbr; sparsity++) { + for (int times = 0; times < 2; ++times) { + String line = ""; + long bef, aft; + line += sparsity; + int[][] data = new int[N][]; + int Max = (1 << (nbr + sparsity)); + for (int k = 0; k < N; ++k) + data[k] = cdg.generateClustered(1 << nbr, Max); + // building + EWAHCompressedBitmap[] ewah = new EWAHCompressedBitmap[N]; + for (int k = 0; k < N; ++k) { + ewah[k] = new EWAHCompressedBitmap(); + for (int x = 0; x < data[k].length; ++x) { + ewah[k].set(data[k][x]); + } + data[k] = null; + } + // sanity check + if (true) { + EWAHCompressedBitmap answer = ewah[0].xor(ewah[1]); + for (int k = 2; k < ewah.length; ++k) + answer = answer.xor(ewah[k]); + EWAHCompressedBitmap ewahor3 = FastAggregation.xor(ewah); + if (!answer.equals(ewahor3)) + throw new RuntimeException("bug FastAggregation.xor"); + EWAHCompressedBitmap ewahor2 = FastAggregation + .bufferedxor(65536,ewah); + if (!answer.equals(ewahor2)) + throw new RuntimeException( + "bug FastAggregation.bufferedxor "); + EWAHCompressedBitmap iwah = IteratorUtil.materialize(IteratorAggregation.bufferedxor(IteratorUtil.toIterators(ewah))); + if (!answer.equals(iwah)) + throw new RuntimeException( + "bug xor it "); + + + } + + // logical xor + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + EWAHCompressedBitmap ewahor = ewah[0]; + for (int j = 1; j < k + 1; ++j) { + ewahor = ewahor.xor(ewah[j]); + } + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + + // fast logical xor + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + EWAHCompressedBitmap[] ewahcp = new EWAHCompressedBitmap[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = ewah[j]; + } + EWAHCompressedBitmap ewahor = FastAggregation + .xor(ewahcp); + bogus += ewahor.sizeInBits(); + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + + + // fast logical xor + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + EWAHCompressedBitmap[] ewahcp = new EWAHCompressedBitmap[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = ewah[j]; + } + EWAHCompressedBitmap ewahor = FastAggregation + .bufferedxor(65536,ewahcp); + bogus += ewahor.sizeInBits(); + } + aft = System.currentTimeMillis(); + line += "\t" + df.format((aft - bef) / 1000.0); + // fast logical xor + bef = System.currentTimeMillis(); + for (int r = 0; r < repeat; ++r) + for (int k = 0; k < N; ++k) { + IteratingRLW[] ewahcp = new IteratingRLW[k + 1]; + for (int j = 0; j < k + 1; ++j) { + ewahcp[j] = new IteratingBufferedRunningLengthWord( + ewah[j]); + } + IteratingRLW ewahor = IteratorAggregation.bufferedxor(ewahcp); + int wordcounter = IteratorUtil.cardinality(ewahor); + bogus += wordcounter; + } + aft = System.currentTimeMillis(); + + line += "\t" + df.format((aft - bef) / 1000.0); + + System.out + .println("# times for: 2by2 FastAggregation.xor bufferedxor iterator-based"); + + System.out.println(line); + } + System.out.println("# bogus =" + bogus); + + } + } +} diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/benchmark/ClusteredDataGenerator.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/benchmark/ClusteredDataGenerator.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/benchmark/ClusteredDataGenerator.java 2013-06-11 17:48:37.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/benchmark/ClusteredDataGenerator.java 2013-11-12 14:31:20.000000000 +0000 @@ -2,23 +2,39 @@ /* - * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc. and Veronika Zenz - * Licensed under APL 2.0. + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. */ /** - * This class will generate uniformly distributed lists of random integers. + * This class will generate lists of random integers with a "clustered" distribution. + * Reference: + * Anh VN, Moffat A. Index compression using 64-bit words. Software: Practice and Experience 2010; 40(2):131-147. * * @author Daniel Lemire */ -class ClusteredDataGenerator { +public class ClusteredDataGenerator { - public ClusteredDataGenerator() { + /** + * + */ +public ClusteredDataGenerator() { + this.unidg = new UniformDataGenerator(); } /** + * @param seed random seed + */ +public ClusteredDataGenerator(final int seed) { + this.unidg = new UniformDataGenerator(seed); +} + +/** * generates randomly N distinct integers from 0 to Max. + * @param N number of integers + * @param Max maximum integer value + * @return a randomly generated array */ public int[] generateClustered(int N, int Max) { int[] array = new int[N]; @@ -56,13 +72,7 @@ array[k + offset] = Min + v[k]; } - public static void main(String[] args) { - int[] example = (new ClusteredDataGenerator()).generateClustered(20, 1000); - for (int k = 0; k < example.length; ++k) - System.out.println(example[k]); - } - - UniformDataGenerator unidg = new UniformDataGenerator(); + UniformDataGenerator unidg; } diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/benchmark/UniformDataGenerator.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/benchmark/UniformDataGenerator.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/benchmark/UniformDataGenerator.java 2013-06-11 17:48:37.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/benchmark/UniformDataGenerator.java 2013-11-12 14:31:20.000000000 +0000 @@ -1,50 +1,114 @@ package com.googlecode.javaewah.benchmark; - /* - * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc. and Veronika Zenz - * Licensed under APL 2.0. + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. */ - import java.util.Arrays; +import java.util.BitSet; import java.util.HashSet; import java.util.Iterator; import java.util.Random; - /** - * This class will generate "clustered" lists of random integers. That is, the - * integers tend not to be randomly distributed. + * This class will generate "uniform" lists of random integers. * * @author Daniel Lemire */ public class UniformDataGenerator { - public UniformDataGenerator(){ + /** + * construct generator of random arrays. + */ + public UniformDataGenerator() { + this.rand = new Random(); + } + + /** + * @param seed random seed + */ + public UniformDataGenerator(final int seed) { + this.rand = new Random(seed); + } + + /** + * generates randomly N distinct integers from 0 to Max. + */ + int[] generateUniformHash(int N, int Max) { + if (N > Max) + throw new RuntimeException("not possible"); + int[] ans = new int[N]; + HashSet s = new HashSet(); + while (s.size() < N) + s.add(new Integer(this.rand.nextInt(Max))); + Iterator i = s.iterator(); + for (int k = 0; k < N; ++k) + ans[k] = i.next().intValue(); + Arrays.sort(ans); + return ans; + } + + /** + * output all integers from the range [0,Max) that are not + * in the array + */ + static int[] negate(int[] x, int Max) { + int[] ans = new int[Max - x.length]; + int i = 0; + int c = 0; + for (int j = 0; j < x.length; ++j) { + int v = x[j]; + for (; i < v; ++i) + ans[c++] = i; + ++i; + } + while (c < ans.length) + ans[c++] = i++; + return ans; + } + + + /** + * generates randomly N distinct integers from 0 to Max. + * @param N Number of integers to generate + * @param Max Maximum value of the integers + * @return array containing random integers + */ + public int[] generateUniform(int N, int Max) { + if(N * 2 > Max) { + return negate( generateUniform(Max - N, Max), Max ); + } + if (2048 * N > Max) + return generateUniformBitmap(N, Max); + return generateUniformHash(N, Max); + } + + /** + * generates randomly N distinct integers from 0 to Max using a bitmap. + * @param N Number of integers to generate + * @param Max Maximum value of the integers + * @return array containing random integers + */ + int[] generateUniformBitmap(int N, int Max) { + if (N > Max) + throw new RuntimeException("not possible"); + int[] ans = new int[N]; + BitSet bs = new BitSet(Max); + int cardinality = 0; + while (cardinality < N) { + int v = this.rand.nextInt(Max); + if (!bs.get(v)) { + bs.set(v); + cardinality++; + } + } + int pos = 0; + for (int i = bs.nextSetBit(0); i >= 0; i = bs.nextSetBit(i + 1)) { + ans[pos++] = i; + } + return ans; } - /** - * generates randomly N distinct integers from 0 to Max. - */ - int[] generateUniform(int N, int Max) { - if (N > Max) - throw new RuntimeException("not possible"); - int[] ans = new int[N]; - if (N == Max) { - for (int k = 0; k < N; ++k) - ans[k]=k; - return ans; - } - HashSet s = new HashSet(); - while (s.size() < N) - s.add(new Integer(this.rand.nextInt(Max))); - Iterator i = s.iterator(); - for (int k = 0; k < N; ++k) - ans[k]=i.next().intValue(); - Arrays.sort(ans); - return ans; - } - Random rand = new Random(); + Random rand = new Random(); - } diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/BitCounter.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/BitCounter.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/BitCounter.java 2013-06-11 17:48:37.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/BitCounter.java 2013-11-12 14:31:20.000000000 +0000 @@ -1,8 +1,8 @@ package com.googlecode.javaewah; /* - * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc. and Veronika Zenz - * Licensed under APL 2.0. + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. */ /** * BitCounter is a fake bitset data structure. Instead of storing the actual @@ -20,7 +20,7 @@ * @param newdata * the word */ - // @Override : causes problems with Java 1.5 + @Override public void add(final long newdata) { this.oneBits += Long.bitCount(newdata); return; @@ -36,7 +36,7 @@ * @param number * the number of literal words to add */ - // @Override : causes problems with Java 1.5 + @Override public void addStreamOfLiteralWords(long[] data, int start, int number) { for (int i = start; i < start + number; i++) { add(data[i]); @@ -51,10 +51,9 @@ * zeros or ones * @param number * how many to words add - * @return the number of words added to the buffer */ - // @Override : causes problems with Java 1.5 - public void addStreamOfEmptyWords(boolean v, long number) { + @Override +public void addStreamOfEmptyWords(boolean v, long number) { if (v) { this.oneBits += number * EWAHCompressedBitmap.wordinbits; } @@ -72,7 +71,8 @@ * the number of literal words to add */ // @Override : causes problems with Java 1.5 - public void addStreamOfNegatedLiteralWords(long[] data, int start, int number) { + @Override +public void addStreamOfNegatedLiteralWords(long[] data, int start, int number) { for (int i = start; i < start + number; i++) { add(~data[i]); } @@ -96,7 +96,8 @@ * number of bits */ // @Override : causes problems with Java 1.5 - public void setSizeInBits(int bits) { + @Override +public void setSizeInBits(int bits) { // no action } diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/BitmapStorage.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/BitmapStorage.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/BitmapStorage.java 2013-06-11 17:48:37.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/BitmapStorage.java 2013-11-12 14:31:20.000000000 +0000 @@ -1,8 +1,8 @@ package com.googlecode.javaewah; /* - * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc. and Veronika Zenz - * Licensed under APL 2.0. + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. */ /** diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/BufferedIterator.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/BufferedIterator.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/BufferedIterator.java 1970-01-01 00:00:00.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/BufferedIterator.java 2013-11-12 14:31:20.000000000 +0000 @@ -0,0 +1,151 @@ +package com.googlecode.javaewah; + + +/* + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. + */ +/** + * This class can be used to iterate over blocks of bitmap data. + * + * @author Daniel Lemire + * + */ +public class BufferedIterator implements IteratingRLW { + /** + * Instantiates a new iterating buffered running length word. + * + * @param iterator iterator + */ + public BufferedIterator(final CloneableIterator iterator) { + this.masteriterator = iterator; + if(this.masteriterator.hasNext()) { + this.iterator = this.masteriterator.next(); + this.brlw = new BufferedRunningLengthWord(this.iterator.next()); + this.literalWordStartPosition = this.iterator.literalWords() + this.brlw.literalwordoffset; + this.buffer = this.iterator.buffer(); + } + } + + + /** + * Discard first words, iterating to the next running length word if needed. + * + * @param x the number of words to be discarded + */ + @Override + public void discardFirstWords(long x) { + while (x > 0) { + if (this.brlw.RunningLength > x) { + this.brlw.RunningLength -= x; + return; + } + x -= this.brlw.RunningLength; + this.brlw.RunningLength = 0; + long toDiscard = x > this.brlw.NumberOfLiteralWords ? this.brlw.NumberOfLiteralWords : x; + + this.literalWordStartPosition += toDiscard; + this.brlw.NumberOfLiteralWords -= toDiscard; + x -= toDiscard; + if ((x > 0) || (this.brlw.size() == 0)) { + if (!this.next()) { + break; + } + } + } + } + /** + * Move to the next RunningLengthWord + * @return whether the move was possible + */ + @Override + public boolean next() { + if (!this.iterator.hasNext()) { + if(!reload()) { + this.brlw.NumberOfLiteralWords = 0; + this.brlw.RunningLength = 0; + return false; + } + } + this.brlw.reset(this.iterator.next()); + this.literalWordStartPosition = this.iterator.literalWords(); // + this.brlw.literalwordoffset ==0 + return true; + } + private boolean reload() { + if(!this.masteriterator.hasNext()) { + return false; + } + this.iterator = this.masteriterator.next(); + this.buffer = this.iterator.buffer(); + return true; + } + + + /** + * Get the nth literal word for the current running length word + * @param index zero based index + * @return the literal word + */ + @Override + public long getLiteralWordAt(int index) { + return this.buffer[this.literalWordStartPosition + index]; + } + + /** + * Gets the number of literal words for the current running length word. + * + * @return the number of literal words + */ + @Override + public int getNumberOfLiteralWords() { + return this.brlw.NumberOfLiteralWords; + } + + /** + * Gets the running bit. + * + * @return the running bit + */ + @Override + public boolean getRunningBit() { + return this.brlw.RunningBit; + } + + /** + * Gets the running length. + * + * @return the running length + */ + @Override + public long getRunningLength() { + return this.brlw.RunningLength; + } + + /** + * Size in uncompressed words of the current running length word. + * + * @return the size + */ + @Override + public long size() { + return this.brlw.size(); + } + + + @Override + public BufferedIterator clone() throws CloneNotSupportedException { + BufferedIterator answer = (BufferedIterator) super.clone(); + answer.brlw = this.brlw.clone(); + answer.buffer = this.buffer; + answer.iterator = this.iterator.clone(); + answer.literalWordStartPosition = this.literalWordStartPosition; + answer.masteriterator = this.masteriterator.clone(); + return answer; + } + + private BufferedRunningLengthWord brlw; + private long[] buffer; + private int literalWordStartPosition; + private EWAHIterator iterator; + private CloneableIterator masteriterator; + } \ No newline at end of file diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/BufferedRunningLengthWord.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/BufferedRunningLengthWord.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/BufferedRunningLengthWord.java 2013-06-11 17:48:37.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/BufferedRunningLengthWord.java 2013-11-12 14:31:20.000000000 +0000 @@ -1,8 +1,8 @@ package com.googlecode.javaewah; /* - * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc. and Veronika Zenz - * Licensed under APL 2.0. + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. */ @@ -15,7 +15,7 @@ * @since 0.1.0 * */ -public final class BufferedRunningLengthWord { +public final class BufferedRunningLengthWord implements Cloneable { /** * Instantiates a new buffered running length word. @@ -34,7 +34,7 @@ * @param rlw the rlw */ public BufferedRunningLengthWord(final RunningLengthWord rlw) { - this(rlw.array[rlw.position]); + this(rlw.parent.buffer[rlw.position]); } /** @@ -99,7 +99,7 @@ * @param rlw the other running length word */ public void reset(final RunningLengthWord rlw) { - reset(rlw.array[rlw.position]); + reset(rlw.parent.buffer[rlw.position]); } /** @@ -147,6 +147,17 @@ + getRunningLength() + " number of lit. words " + getNumberOfLiteralWords(); } + + @Override +public BufferedRunningLengthWord clone() throws CloneNotSupportedException { + BufferedRunningLengthWord answer = (BufferedRunningLengthWord) super.clone(); + answer.literalwordoffset = this.literalwordoffset; + answer.NumberOfLiteralWords = this.NumberOfLiteralWords; + answer.RunningBit = this.RunningBit; + answer.RunningLength = this.RunningLength; + return answer; + } + /** how many literal words have we read so far? */ public int literalwordoffset = 0; diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/CloneableIterator.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/CloneableIterator.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/CloneableIterator.java 1970-01-01 00:00:00.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/CloneableIterator.java 2013-11-12 14:31:20.000000000 +0000 @@ -0,0 +1,24 @@ +package com.googlecode.javaewah; + +/** + * Like a standard Java iterator, except that you can clone it. + * + * @param the data type of the iterator + */ +public interface CloneableIterator extends Cloneable { + + /** + * @return whether there is more + */ + public boolean hasNext(); + /** + * @return the next element + */ + public E next(); + /** + * @return a copy + * @throws CloneNotSupportedException this should never happen in practice + */ + public CloneableIterator clone() throws CloneNotSupportedException; + +} \ No newline at end of file diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/EWAHCompressedBitmap.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/EWAHCompressedBitmap.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/EWAHCompressedBitmap.java 2013-06-11 17:48:37.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/EWAHCompressedBitmap.java 2013-11-12 14:31:20.000000000 +0000 @@ -1,14 +1,15 @@ package com.googlecode.javaewah; /* - * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc. and Veronika Zenz - * Licensed under APL 2.0. + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. */ import java.util.*; import java.io.*; + /** *

* This implements the patent-free(1) EWAH scheme. Roughly speaking, it is a @@ -41,7 +42,7 @@ * *

    *
  • Daniel Lemire, Owen Kaser, Kamel Aouiche, Sorting improves - * word-aligned bitmap indexes. Data & Knowledge Engineering 69 (1), pages + * word-aligned bitmap indexes. Data & Knowledge Engineering 69 (1), pages * 3-28, 2010. http://arxiv.org/abs/0901.3751
    • *
* @@ -81,7 +82,7 @@ */ public EWAHCompressedBitmap() { this.buffer = new long[defaultbuffersize]; - this.rlw = new RunningLengthWord(this.buffer, 0); + this.rlw = new RunningLengthWord(this, 0); } /** @@ -94,7 +95,7 @@ */ public EWAHCompressedBitmap(final int buffersize) { this.buffer = new long[buffersize]; - this.rlw = new RunningLengthWord(this.buffer, 0); + this.rlw = new RunningLengthWord(this, 0); } /** @@ -103,10 +104,15 @@ * This is normally how you add data to the array. So you add bits in streams * of 8*8 bits. * + * Example: if you add 321, you are have added (in binary notation) + * 0b101000001, so you have effectively called set(0), set(6), set(8) + * in sequence. + * * @param newdata * the word */ - public void add(final long newdata) { + @Override +public void add(final long newdata) { add(newdata, wordinbits); } @@ -182,13 +188,14 @@ * @param number * the number of literal words to add */ - public void addStreamOfLiteralWords(final long[] data, final int start, + @Override +public void addStreamOfLiteralWords(final long[] data, final int start, final int number) { int leftovernumber = number; while(leftovernumber > 0) { final int NumberOfLiteralWords = this.rlw.getNumberOfLiteralWords(); - final int whatwecanadd = number < RunningLengthWord.largestliteralcount - - NumberOfLiteralWords ? number : RunningLengthWord.largestliteralcount + final int whatwecanadd = leftovernumber < RunningLengthWord.largestliteralcount + - NumberOfLiteralWords ? leftovernumber : RunningLengthWord.largestliteralcount - NumberOfLiteralWords; this.rlw.setNumberOfLiteralWords(NumberOfLiteralWords + whatwecanadd); leftovernumber -= whatwecanadd; @@ -210,7 +217,8 @@ * @param number * the number */ - public void addStreamOfEmptyWords(final boolean v, long number) { + @Override +public void addStreamOfEmptyWords(final boolean v, long number) { if (number == 0) return; this.sizeinbits += number * wordinbits; @@ -255,13 +263,14 @@ * @param number * the number of literal words to add */ + @Override public void addStreamOfNegatedLiteralWords(final long[] data, final int start, final int number) { int leftovernumber = number; while (leftovernumber > 0) { final int NumberOfLiteralWords = this.rlw.getNumberOfLiteralWords(); - final int whatwecanadd = number < RunningLengthWord.largestliteralcount - - NumberOfLiteralWords ? number + final int whatwecanadd = leftovernumber < RunningLengthWord.largestliteralcount + - NumberOfLiteralWords ? leftovernumber : RunningLengthWord.largestliteralcount - NumberOfLiteralWords; this.rlw.setNumberOfLiteralWords(NumberOfLiteralWords @@ -283,12 +292,16 @@ * The running time is proportional to the sum of the compressed sizes (as * reported by sizeInBytes()). * + * If you are not planning on adding to the resulting bitmap, you may call the trim() + * method to reduce memory usage. + * * @since 0.4.3 * @param a * the other bitmap * @return the EWAH compressed bitmap */ - public EWAHCompressedBitmap and(final EWAHCompressedBitmap a) { + @Override +public EWAHCompressedBitmap and(final EWAHCompressedBitmap a) { final EWAHCompressedBitmap container = new EWAHCompressedBitmap(); container .reserve(this.actualsizeinwords > a.actualsizeinwords ? this.actualsizeinwords @@ -341,10 +354,12 @@ rlwj.discardFirstWords(nbre_literal); } } - final boolean i_remains = rlwi.size()>0; - final IteratingBufferedRunningLengthWord remaining = i_remains ? rlwi : rlwj; - remaining.dischargeAsEmpty(container); - container.setSizeInBits(Math.max(sizeInBits(), a.sizeInBits())); + if(adjustContainerSizeWhenAggregating) { + final boolean i_remains = rlwi.size()>0; + final IteratingBufferedRunningLengthWord remaining = i_remains ? rlwi : rlwj; + remaining.dischargeAsEmpty(container); + container.setSizeInBits(Math.max(sizeInBits(), a.sizeInBits())); + } } @@ -371,11 +386,15 @@ * The running time is proportional to the sum of the compressed sizes (as * reported by sizeInBytes()). * + * If you are not planning on adding to the resulting bitmap, you may call the trim() + * method to reduce memory usage. + * * @param a * the other bitmap * @return the EWAH compressed bitmap */ - public EWAHCompressedBitmap andNot(final EWAHCompressedBitmap a) { + @Override +public EWAHCompressedBitmap andNot(final EWAHCompressedBitmap a) { final EWAHCompressedBitmap container = new EWAHCompressedBitmap(); container .reserve(this.actualsizeinwords > a.actualsizeinwords ? this.actualsizeinwords @@ -387,15 +406,14 @@ /** * Returns a new compressed bitmap containing the bitwise AND NOT values of * the current bitmap with some other bitmap. This method is expected to - * be faster than doing A.and(((EWAHCompressedBitmap) B.clone()).not()). + * be faster than doing A.and(B.clone().not()). * * The running time is proportional to the sum of the compressed sizes (as * reported by sizeInBytes()). * * @since 0.4.0 - * @param a - * the other bitmap - * @return the EWAH compressed bitmap + * @param a the other bitmap + * @param container where to store the result */ public void andNotToContainer(final EWAHCompressedBitmap a, final BitmapStorage container) { @@ -440,9 +458,10 @@ final IteratingBufferedRunningLengthWord remaining = i_remains ? rlwi : rlwj; if(i_remains) remaining.discharge(container); - else + else if(adjustContainerSizeWhenAggregating) remaining.dischargeAsEmpty(container); - container.setSizeInBits(Math.max(sizeInBits(), a.sizeInBits())); + if(adjustContainerSizeWhenAggregating) + container.setSizeInBits(Math.max(sizeInBits(), a.sizeInBits())); } /** @@ -469,7 +488,7 @@ */ public int cardinality() { int counter = 0; - final EWAHIterator i = new EWAHIterator(this.buffer, this.actualsizeinwords); + final EWAHIterator i = new EWAHIterator(this, this.actualsizeinwords); while (i.hasNext()) { RunningLengthWord localrlw = i.next(); if (localrlw.getRunningBit()) { @@ -498,10 +517,10 @@ * @see java.lang.Object#clone() */ @Override - public Object clone() throws java.lang.CloneNotSupportedException { + public EWAHCompressedBitmap clone() throws java.lang.CloneNotSupportedException { final EWAHCompressedBitmap clone = (EWAHCompressedBitmap) super.clone(); clone.buffer = this.buffer.clone(); - clone.rlw = new RunningLengthWord(clone.buffer, this.rlw.position); + clone.rlw = new RunningLengthWord(clone, this.rlw.position); clone.actualsizeinwords = this.actualsizeinwords; clone.sizeinbits = this.sizeinbits; return clone; @@ -523,13 +542,12 @@ } for (int k = 0; k < this.actualsizeinwords; ++k) this.buffer[k] = in.readLong(); - this.rlw = new RunningLengthWord(this.buffer, in.readInt()); + this.rlw = new RunningLengthWord(this, in.readInt()); } /** * Check to see whether the two compressed bitmaps contain the same set bits. * - * @author Colby Ranger * @see java.lang.Object#equals(java.lang.Object) */ @Override @@ -554,7 +572,6 @@ * the boolean value * @param number * the number (must be greater than 0) - * @return nothing */ private void fastaddStreamOfEmptyWords(final boolean v, long number) { if ((this.rlw.getRunningBit() != v) && (this.rlw.size() == 0)) { @@ -597,10 +614,16 @@ * @return the EWAHIterator */ public EWAHIterator getEWAHIterator() { - return new EWAHIterator(this.buffer, this.actualsizeinwords); + return new EWAHIterator(this, this.actualsizeinwords); } /** + * @return the IteratingRLW iterator corresponding to this bitmap + */ + public IteratingRLW getIteratingRLW() { + return new IteratingBufferedRunningLengthWord(this); + } + /** * get the locations of the true values as one vector. (may use more memory * than iterator()) * @@ -608,7 +631,7 @@ */ public List getPositions() { final ArrayList v = new ArrayList(); - final EWAHIterator i = new EWAHIterator(this.buffer, this.actualsizeinwords); + final EWAHIterator i = new EWAHIterator(this, this.actualsizeinwords); int pos = 0; while (i.hasNext()) { RunningLengthWord localrlw = i.next(); @@ -645,7 +668,7 @@ public int hashCode() { int karprabin = 0; final int B = 31; - final EWAHIterator i = new EWAHIterator(this.buffer, this.actualsizeinwords); + final EWAHIterator i = new EWAHIterator(this, this.actualsizeinwords); while( i.hasNext() ) { i.next(); if (i.rlw.getRunningBit() == true) { @@ -691,7 +714,7 @@ */ public IntIterator intIterator() { return new IntIteratorImpl( - new EWAHIterator(this.buffer, this.actualsizeinwords)); + new EWAHIterator(this, this.actualsizeinwords)); } /** @@ -700,17 +723,21 @@ * * @return the iterator */ - public Iterator iterator() { + @Override +public Iterator iterator() { return new Iterator() { - public boolean hasNext() { + @Override + public boolean hasNext() { return this.under.hasNext(); } - public Integer next() { + @Override + public Integer next() { return new Integer(this.under.next()); } - public void remove() { + @Override + public void remove() { throw new UnsupportedOperationException("bitsets do not support remove"); } @@ -732,9 +759,14 @@ final int number) { while (this.actualsizeinwords + number >= this.buffer.length) { final long oldbuffer[] = this.buffer; - this.buffer = new long[oldbuffer.length * 2]; + if((this.actualsizeinwords + number) < 32768) + this.buffer = new long[ (this.actualsizeinwords + number) * 2]; + else if((this.actualsizeinwords + number) * 3 / 2 < this.actualsizeinwords + number) // overflow + this.buffer = new long[Integer.MAX_VALUE]; + else + this.buffer = new long[(this.actualsizeinwords + number) * 3 / 2]; System.arraycopy(oldbuffer, 0, this.buffer, 0, oldbuffer.length); - this.rlw.array = this.buffer; + this.rlw.parent.buffer = this.buffer; } for (int k = 0; k < number; ++k) this.buffer[this.actualsizeinwords + k] = ~data[start + k]; @@ -749,8 +781,9 @@ * sizeInBytes()). * */ - public void not() { - final EWAHIterator i = new EWAHIterator(this.buffer, this.actualsizeinwords); + @Override +public void not() { + final EWAHIterator i = new EWAHIterator(this, this.actualsizeinwords); if (!i.hasNext()) return; @@ -786,17 +819,23 @@ * The running time is proportional to the sum of the compressed sizes (as * reported by sizeInBytes()). * + * If you are not planning on adding to the resulting bitmap, you may call the trim() + * method to reduce memory usage. + * * @param a * the other bitmap * @return the EWAH compressed bitmap */ - public EWAHCompressedBitmap or(final EWAHCompressedBitmap a) { + @Override +public EWAHCompressedBitmap or(final EWAHCompressedBitmap a) { final EWAHCompressedBitmap container = new EWAHCompressedBitmap(); container.reserve(this.actualsizeinwords + a.actualsizeinwords); orToContainer(a, container); return container; } + + /** * Computes the bitwise or between the current bitmap and the bitmap "a". * Stores the result in the container. @@ -872,9 +911,14 @@ private void push_back(final long data) { if (this.actualsizeinwords == this.buffer.length) { final long oldbuffer[] = this.buffer; - this.buffer = new long[oldbuffer.length * 2]; + if(oldbuffer.length < 32768) + this.buffer = new long[ oldbuffer.length * 2]; + else if(oldbuffer.length * 3 / 2 < oldbuffer.length) // overflow + this.buffer = new long[Integer.MAX_VALUE]; + else + this.buffer = new long[oldbuffer.length * 3 / 2]; System.arraycopy(oldbuffer, 0, this.buffer, 0, oldbuffer.length); - this.rlw.array = this.buffer; + this.rlw.parent.buffer = this.buffer; } this.buffer[this.actualsizeinwords++] = data; } @@ -890,11 +934,16 @@ * the number of words to add */ private void push_back(final long[] data, final int start, final int number) { - while (this.actualsizeinwords + number >= this.buffer.length) { + if (this.actualsizeinwords + number >= this.buffer.length) { final long oldbuffer[] = this.buffer; - this.buffer = new long[oldbuffer.length * 2]; + if(this.actualsizeinwords + number < 32768) + this.buffer = new long[(this.actualsizeinwords + number) * 2]; + else if ((this.actualsizeinwords + number) * 3 / 2 < this.actualsizeinwords + number) // overflow + this.buffer = new long[Integer.MAX_VALUE]; + else + this.buffer = new long[( this.actualsizeinwords + number) * 3 / 2]; System.arraycopy(oldbuffer, 0, this.buffer, 0, oldbuffer.length); - this.rlw.array = this.buffer; + this.rlw.parent.buffer = this.buffer; } System.arraycopy(data, start, this.buffer, this.actualsizeinwords, number); this.actualsizeinwords += number; @@ -903,7 +952,8 @@ /* * @see java.io.Externalizable#readExternal(java.io.ObjectInput) */ - public void readExternal(ObjectInput in) throws IOException { + @Override +public void readExternal(ObjectInput in) throws IOException { deserialize(in); } @@ -919,7 +969,7 @@ final long oldbuffer[] = this.buffer; this.buffer = new long[size]; System.arraycopy(oldbuffer, 0, this.buffer, 0, oldbuffer.length); - this.rlw.array = this.buffer; + this.rlw.parent.buffer = this.buffer; return true; } return false; @@ -949,15 +999,48 @@ public int serializedSizeInBytes() { return this.sizeInBytes() + 3 * 4; } + + + /** + * Query the value of a single bit. Relying on this method when speed is + * needed is discouraged. The complexity is linear with the size of the + * bitmap. + * + * (This implementation is based on zhenjl's Go version of JavaEWAH.) + * + * @param i + * the bit we are interested in + * @return whether the bit is set to true + */ + public boolean get(final int i) { + if ((i < 0) || (i >= this.sizeinbits)) + return false; + int WordChecked = 0; + final IteratingRLW j = getIteratingRLW(); + final int wordi = i/wordinbits; + while (WordChecked <= wordi ) { + WordChecked += j.getRunningLength(); + if (wordi < WordChecked) { + return j.getRunningBit(); + } + if (wordi < WordChecked + j.getNumberOfLiteralWords()) { + final long w = j.getLiteralWordAt(wordi - WordChecked); + return (w & (1l << i)) != 0; + } + WordChecked += j.getNumberOfLiteralWords(); + j.next(); + } + return false; + } /** - * set the bit at position i to true, the bits must be set in increasing + * Set the bit at position i to true, the bits must be set in (strictly) increasing * order. For example, set(15) and then set(7) will fail. You must do set(7) * and then set(15). * * @param i * the index - * @return true if the value was set (always true when i>= sizeInBits()). + * @return true if the value was set (always true when i greater or equal to sizeInBits()). * @throws IndexOutOfBoundsException * if i is negative or greater than Integer.MAX_VALUE - 64 */ @@ -994,17 +1077,21 @@ } /** - * set the size in bits + * Set the size in bits. This does not change the compressed bitmap. * * @since 0.4.0 */ - public void setSizeInBits(final int size) { + @Override +public void setSizeInBits(final int size) { + if((size+EWAHCompressedBitmap.wordinbits-1)/EWAHCompressedBitmap.wordinbits!= (this.sizeinbits+EWAHCompressedBitmap.wordinbits-1)/EWAHCompressedBitmap.wordinbits) + throw new RuntimeException("You can only reduce the size of the bitmap within the scope of the last word. To extend the bitmap, please call setSizeInbits(int,boolean)."); this.sizeinbits = size; } /** * Change the reported size in bits of the *uncompressed* bitmap represented - * by this compressed bitmap. It is not possible to reduce the sizeInBits, but + * by this compressed bitmap. It may change the underlying compressed bitmap. + * It is not possible to reduce the sizeInBits, but * it can be extended. The new bits are set to false or true depending on the * value of defaultvalue. * @@ -1042,7 +1129,8 @@ * * @return the size in bits */ - public int sizeInBits() { + @Override +public int sizeInBits() { return this.sizeinbits; } @@ -1052,7 +1140,8 @@ * * @return the size in bytes */ - public int sizeInBytes() { + @Override +public int sizeInBytes() { return this.actualsizeinwords * (wordinbits / 8); } @@ -1066,7 +1155,7 @@ int[] ans = new int[this.cardinality()]; int inanspos = 0; int pos = 0; - final EWAHIterator i = new EWAHIterator(this.buffer, this.actualsizeinwords); + final EWAHIterator i = new EWAHIterator(this, this.actualsizeinwords); while (i.hasNext()) { RunningLengthWord localrlw = i.next(); if (localrlw.getRunningBit()) { @@ -1108,7 +1197,7 @@ public String toDebugString() { String ans = " EWAHCompressedBitmap, size in bits = " + this.sizeinbits + " size in words = " + this.actualsizeinwords + "\n"; - final EWAHIterator i = new EWAHIterator(this.buffer, this.actualsizeinwords); + final EWAHIterator i = new EWAHIterator(this, this.actualsizeinwords); while (i.hasNext()) { RunningLengthWord localrlw = i.next(); if (localrlw.getRunningBit()) { @@ -1144,11 +1233,43 @@ answer.append("}"); return answer.toString(); } + + /** + * swap the content of the bitmap with another. + * @param other bitmap to swap with + */ +public void swap(final EWAHCompressedBitmap other) { + long[] tmp = this.buffer; + this.buffer = other.buffer; + other.buffer = tmp; + + + int tmp2 = this.rlw.position; + this.rlw.position = other.rlw.position; + other.rlw.position = tmp2; + + int tmp3 = this.actualsizeinwords; + this.actualsizeinwords = other.actualsizeinwords; + other.actualsizeinwords = tmp3; + + int tmp4 = this.sizeinbits; + this.sizeinbits = other.sizeinbits; + other.sizeinbits = tmp4; + } + + /** + * Reduce the internal buffer to its minimal allowable size (given + * by this.actualsizeinwords). This can free memory. + */ + public void trim() { + this.buffer = Arrays.copyOf(this.buffer, this.actualsizeinwords); + } /* * @see java.io.Externalizable#writeExternal(java.io.ObjectOutput) */ - public void writeExternal(ObjectOutput out) throws IOException { + @Override +public void writeExternal(ObjectOutput out) throws IOException { serialize(out); } @@ -1159,11 +1280,15 @@ * The running time is proportional to the sum of the compressed sizes (as * reported by sizeInBytes()). * + * If you are not planning on adding to the resulting bitmap, you may call the trim() + * method to reduce memory usage. + * * @param a * the other bitmap * @return the EWAH compressed bitmap */ - public EWAHCompressedBitmap xor(final EWAHCompressedBitmap a) { + @Override +public EWAHCompressedBitmap xor(final EWAHCompressedBitmap a) { final EWAHCompressedBitmap container = new EWAHCompressedBitmap(); container.reserve(this.actualsizeinwords + a.actualsizeinwords); xorToContainer(a, container); @@ -1250,121 +1375,20 @@ */ public static void andWithContainer(final BitmapStorage container, final EWAHCompressedBitmap... bitmaps) { - if (bitmaps.length == 2) { - // should be more efficient - bitmaps[0].andToContainer(bitmaps[1], container); - return; - } - - // Sort the bitmaps in ascending order by sizeinbits. When we exhaust the - // first bitmap the rest - // of the result is zeros. - final EWAHCompressedBitmap[] sortedBitmaps = bitmaps.clone(); - Arrays.sort(sortedBitmaps, new Comparator() { - public int compare(EWAHCompressedBitmap a, EWAHCompressedBitmap b) { - return a.sizeinbits < b.sizeinbits ? -1 - : a.sizeinbits == b.sizeinbits ? 0 : 1; - } - }); - - int maxSize = sortedBitmaps[sortedBitmaps.length - 1].sizeinbits; - - final IteratingBufferedRunningLengthWord[] rlws = new IteratingBufferedRunningLengthWord[bitmaps.length]; - for (int i = 0; i < sortedBitmaps.length; i++) { - EWAHIterator iterator = sortedBitmaps[i].getEWAHIterator(); - if (iterator.hasNext()) { - rlws[i] = new IteratingBufferedRunningLengthWord(iterator); - } else { - // this never happens... - if (maxSize > 0) { - extendEmptyBits(container, 0, maxSize); - } - container.setSizeInBits(maxSize); - return; - } - } - - while (true) { - long maxZeroRl = 0; - long minOneRl = Long.MAX_VALUE; - long minSize = Long.MAX_VALUE; - int numEmptyRl = 0; - - - for (IteratingBufferedRunningLengthWord rlw : rlws) { - - long size = rlw.size(); - minSize = Math.min(minSize, size); - - if (!rlw.getRunningBit()) { - long rl = rlw.getRunningLength(); - maxZeroRl = Math.max(maxZeroRl, rl); - minOneRl = 0; - if (rl == 0 && size > 0) { - numEmptyRl++; - } - } else { - long rl = rlw.getRunningLength(); - minOneRl = Math.min(minOneRl, rl); - if (rl == 0 && size > 0) { - numEmptyRl++; - } - } - } - - if (minSize == 0) { - extendEmptyBits(container, sortedBitmaps[0].sizeinbits, maxSize); - break; - } - if (maxZeroRl > 0) { - container.addStreamOfEmptyWords(false, maxZeroRl); - for (IteratingBufferedRunningLengthWord rlw : rlws) { - rlw.discardFirstWords(maxZeroRl); - } - } else if (minOneRl > 0) { - container.addStreamOfEmptyWords(true, minOneRl); - for (IteratingBufferedRunningLengthWord rlw : rlws) { - rlw.discardFirstWords(minOneRl); - } - } else { - int index = 0; - - if (numEmptyRl == 1) { - // if one rlw has literal words to process and the rest have a run of - // 1's we can write them out here - IteratingBufferedRunningLengthWord emptyRl = null; - long minNonEmptyRl = Long.MAX_VALUE; - for (IteratingBufferedRunningLengthWord rlw : rlws) { - long rl = rlw.getRunningLength(); - if (rl == 0) { - assert emptyRl == null; - emptyRl = rlw; - } else { - minNonEmptyRl = Math.min(minNonEmptyRl, rl); - } - } - long wordsToWrite = minNonEmptyRl > minSize ? minSize : minNonEmptyRl; - if (emptyRl != null) - emptyRl.writeLiteralWords((int) wordsToWrite, container); - index += wordsToWrite; - } - - while (index < minSize) { - long word = ~0l; - for (IteratingBufferedRunningLengthWord rlw : rlws) { - if (rlw.getRunningLength() <= index) { - word &= rlw.getLiteralWordAt(index - (int) rlw.getRunningLength()); - } - } - container.add(word); - index++; - } - for (IteratingBufferedRunningLengthWord rlw : rlws) { - rlw.discardFirstWords(minSize); - } - } - } - container.setSizeInBits(maxSize); + if(bitmaps.length == 1) throw new IllegalArgumentException("Need at least one bitmap"); + if(bitmaps.length == 2) { + bitmaps[0].andToContainer(bitmaps[1],container); + return; + } + EWAHCompressedBitmap answer = new EWAHCompressedBitmap(); + EWAHCompressedBitmap tmp = new EWAHCompressedBitmap(); + bitmaps[0].andToContainer(bitmaps[1], answer); + for(int k = 2; k < bitmaps.length - 1; ++k) { + answer.andToContainer(bitmaps[k], tmp); + tmp.swap(answer); + tmp.clear(); + } + answer.andToContainer(bitmaps[bitmaps.length - 1], container); } /** @@ -1373,20 +1397,28 @@ * * It may or may not be faster than doing the aggregation two-by-two (A.and(B).and(C)). * + * If only one bitmap is provided, it is returned as is. + * + * If you are not planning on adding to the resulting bitmap, you may call the trim() + * method to reduce memory usage. + * * @since 0.4.3 * @param bitmaps * bitmaps to AND together * @return result of the AND */ public static EWAHCompressedBitmap and(final EWAHCompressedBitmap... bitmaps) { - final EWAHCompressedBitmap container = new EWAHCompressedBitmap(); - int largestSize = 0; - for (EWAHCompressedBitmap bitmap : bitmaps) { - largestSize = Math.max(bitmap.actualsizeinwords, largestSize); - } - container.reserve((int) (largestSize * 1.5)); - andWithContainer(container, bitmaps); - return container; + if(bitmaps.length == 1) return bitmaps[0]; + if(bitmaps.length == 2) return bitmaps[0].and(bitmaps[1]); + EWAHCompressedBitmap answer = new EWAHCompressedBitmap(); + EWAHCompressedBitmap tmp = new EWAHCompressedBitmap(); + bitmaps[0].andToContainer(bitmaps[1], answer); + for(int k = 2; k < bitmaps.length; ++k) { + answer.andToContainer(bitmaps[k], tmp); + tmp.swap(answer); + tmp.clear(); + } + return answer; } /** @@ -1400,6 +1432,7 @@ * @return the cardinality */ public static int andCardinality(final EWAHCompressedBitmap... bitmaps) { + if(bitmaps.length == 1) return bitmaps[0].cardinality(); final BitCounter counter = new BitCounter(); andWithContainer(counter, bitmaps); return counter.getCount(); @@ -1446,152 +1479,75 @@ + (currentLeftover != 0 ? -1 : 0)); } - /** - * For internal use. Computes the bitwise or of the provided bitmaps and - * stores the result in the container. - * - * @since 0.4.0 - */ - public static void orWithContainer(final BitmapStorage container, - final EWAHCompressedBitmap... bitmaps) { - if (bitmaps.length == 2) { - // should be more efficient - bitmaps[0].orToContainer(bitmaps[1], container); - return; - } - - // Sort the bitmaps in descending order by sizeinbits. We will exhaust the - // sorted bitmaps from right to left. - final EWAHCompressedBitmap[] sortedBitmaps = bitmaps.clone(); - Arrays.sort(sortedBitmaps, new Comparator() { - public int compare(EWAHCompressedBitmap a, EWAHCompressedBitmap b) { - return a.sizeinbits < b.sizeinbits ? 1 - : a.sizeinbits == b.sizeinbits ? 0 : -1; - } - }); - - final IteratingBufferedRunningLengthWord[] rlws = new IteratingBufferedRunningLengthWord[bitmaps.length]; - int maxAvailablePos = 0; - for (EWAHCompressedBitmap bitmap : sortedBitmaps) { - EWAHIterator iterator = bitmap.getEWAHIterator(); - if (iterator.hasNext()) { - rlws[maxAvailablePos++] = new IteratingBufferedRunningLengthWord( - iterator); - } - } - - if (maxAvailablePos == 0) { // this never happens... - container.setSizeInBits(0); - return; - } - - int maxSize = sortedBitmaps[0].sizeinbits; - - while (true) { - long maxOneRl = 0; - long minZeroRl = Long.MAX_VALUE; - long minSize = Long.MAX_VALUE; - int numEmptyRl = 0; - for (int i = 0; i < maxAvailablePos; i++) { - IteratingBufferedRunningLengthWord rlw = rlws[i]; - long size = rlw.size(); - if (size == 0) { - maxAvailablePos = i; - break; - } - minSize = Math.min(minSize, size); - - if (rlw.getRunningBit()) { - long rl = rlw.getRunningLength(); - maxOneRl = Math.max(maxOneRl, rl); - minZeroRl = 0; - if (rl == 0 && size > 0) { - numEmptyRl++; - } - } else { - long rl = rlw.getRunningLength(); - minZeroRl = Math.min(minZeroRl, rl); - if (rl == 0 && size > 0) { - numEmptyRl++; - } - } - } - - if (maxAvailablePos == 0) { - break; - } else if (maxAvailablePos == 1) { - // only one bitmap is left so just write the rest of it out - rlws[0].discharge(container); - break; - } - - if (maxOneRl > 0) { - container.addStreamOfEmptyWords(true, maxOneRl); - for (int i = 0; i < maxAvailablePos; i++) { - IteratingBufferedRunningLengthWord rlw = rlws[i]; - rlw.discardFirstWords(maxOneRl); - } - } else if (minZeroRl > 0) { - container.addStreamOfEmptyWords(false, minZeroRl); - for (int i = 0; i < maxAvailablePos; i++) { - IteratingBufferedRunningLengthWord rlw = rlws[i]; - rlw.discardFirstWords(minZeroRl); - } - } else { - int index = 0; - - if (numEmptyRl == 1) { - // if one rlw has literal words to process and the rest have a run of - // 0's we can write them out here - IteratingBufferedRunningLengthWord emptyRl = null; - long minNonEmptyRl = Long.MAX_VALUE; - for (int i = 0; i < maxAvailablePos; i++) { - IteratingBufferedRunningLengthWord rlw = rlws[i]; - long rl = rlw.getRunningLength(); - if (rl == 0) { - assert emptyRl == null; - emptyRl = rlw; - } else { - minNonEmptyRl = Math.min(minNonEmptyRl, rl); - } - } - long wordsToWrite = minNonEmptyRl > minSize ? minSize : minNonEmptyRl; - if (emptyRl != null) - emptyRl.writeLiteralWords((int) wordsToWrite, container); - index += wordsToWrite; - } - - while (index < minSize) { - long word = 0; - for (int i = 0; i < maxAvailablePos; i++) { - IteratingBufferedRunningLengthWord rlw = rlws[i]; - if (rlw.getRunningLength() <= index) { - word |= rlw.getLiteralWordAt(index - (int) rlw.getRunningLength()); - } - } - container.add(word); - index++; - } - for (int i = 0; i < maxAvailablePos; i++) { - IteratingBufferedRunningLengthWord rlw = rlws[i]; - rlw.discardFirstWords(minSize); - } - } - } - container.setSizeInBits(maxSize); - } + + /** + * Uses an adaptive technique to compute the logical OR. + * Mostly for internal use. + * + * @param container where the aggregate is written. + * @param bitmaps to be aggregated + */ + public static void orWithContainer(final BitmapStorage container, + final EWAHCompressedBitmap... bitmaps) { + if (bitmaps.length < 2) + throw new IllegalArgumentException("You should provide at least two bitmaps, provided "+bitmaps.length); + long size = 0L; + long sinbits = 0L; + for (EWAHCompressedBitmap b : bitmaps) { + size += b.sizeInBytes(); + if (sinbits < b.sizeInBits()) + sinbits = b.sizeInBits(); + } + if (size * 8 > sinbits) { + FastAggregation.bufferedorWithContainer(container, 65536, bitmaps); + } else { + FastAggregation.orToContainer(container, bitmaps); + } + } + + /** + * Uses an adaptive technique to compute the logical XOR. + * Mostly for internal use. + * + * @param container where the aggregate is written. + * @param bitmaps to be aggregated + */ + public static void xorWithContainer(final BitmapStorage container, + final EWAHCompressedBitmap... bitmaps) { + if (bitmaps.length < 2) + throw new IllegalArgumentException("You should provide at least two bitmaps, provided "+bitmaps.length); + long size = 0L; + long sinbits = 0L; + for (EWAHCompressedBitmap b : bitmaps) { + size += b.sizeInBytes(); + if (sinbits < b.sizeInBits()) + sinbits = b.sizeInBits(); + } + if (size * 8 > sinbits) { + FastAggregation.bufferedxorWithContainer(container, 65536, bitmaps); + } else { + FastAggregation.xorToContainer(container, bitmaps); + } + } /** * Returns a new compressed bitmap containing the bitwise OR values of the * provided bitmaps. This is typically faster than doing the aggregation * two-by-two (A.or(B).or(C).or(D)). * + * If only one bitmap is provided, it is returned as is. + * + * If you are not planning on adding to the resulting bitmap, you may call the trim() + * method to reduce memory usage. + * * @since 0.4.0 * @param bitmaps * bitmaps to OR together * @return result of the OR */ public static EWAHCompressedBitmap or(final EWAHCompressedBitmap... bitmaps) { + if(bitmaps.length == 1) + return bitmaps[0]; final EWAHCompressedBitmap container = new EWAHCompressedBitmap(); int largestSize = 0; for (EWAHCompressedBitmap bitmap : bitmaps) { @@ -1601,6 +1557,32 @@ orWithContainer(container, bitmaps); return container; } + /** + * Returns a new compressed bitmap containing the bitwise XOR values of the + * provided bitmaps. This is typically faster than doing the aggregation + * two-by-two (A.xor(B).xor(C).xor(D)). + * + * If only one bitmap is provided, it is returned as is. + * + * If you are not planning on adding to the resulting bitmap, you may call the trim() + * method to reduce memory usage. + * + * @param bitmaps + * bitmaps to XOR together + * @return result of the XOR + */ + public static EWAHCompressedBitmap xor(final EWAHCompressedBitmap... bitmaps) { + if(bitmaps.length == 1) + return bitmaps[0]; + final EWAHCompressedBitmap container = new EWAHCompressedBitmap(); + int largestSize = 0; + for (EWAHCompressedBitmap bitmap : bitmaps) { + largestSize = Math.max(bitmap.actualsizeinwords, largestSize); + } + container.reserve((int) (largestSize * 1.5)); + xorWithContainer(container, bitmaps); + return container; + } /** * Returns the cardinality of the result of a bitwise OR of the values of the @@ -1613,6 +1595,7 @@ * @return the cardinality */ public static int orCardinality(final EWAHCompressedBitmap... bitmaps) { + if(bitmaps.length == 1) return bitmaps[0].cardinality(); final BitCounter counter = new BitCounter(); orWithContainer(counter, bitmaps); return counter.getCount(); @@ -1638,6 +1621,9 @@ /** optimization option **/ public static final boolean usetrailingzeros = true; + + /** whether we adjust after some aggregation by adding in zeroes **/ + public static final boolean adjustContainerSizeWhenAggregating = true; /** The Constant wordinbits represents the number of bits in a long. */ public static final int wordinbits = 64; diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/EWAHIterator.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/EWAHIterator.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/EWAHIterator.java 2013-06-11 17:48:37.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/EWAHIterator.java 2013-11-12 14:31:20.000000000 +0000 @@ -2,27 +2,27 @@ /* - * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc. and Veronika Zenz - * Licensed under APL 2.0. + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. */ /** * The class EWAHIterator represents a special type of * efficient iterator iterating over (uncompressed) words of bits. - * + * It is not meant for end users. * @author Daniel Lemire * @since 0.1.0 * */ -public final class EWAHIterator { +public final class EWAHIterator implements Cloneable { /** - * Instantiates a new eWAH iterator. + * Instantiates a new EWAH iterator. * * @param a the array of words * @param sizeinwords the number of words that are significant in the array of words */ - public EWAHIterator(final long[] a, final int sizeinwords) { + public EWAHIterator(final EWAHCompressedBitmap a, final int sizeinwords) { this.rlw = new RunningLengthWord(a, 0); this.size = sizeinwords; this.pointer = 0; @@ -31,7 +31,7 @@ /** * Allow expert developers to instantiate an EWAHIterator. * - * @param bitmap + * @param bitmap we want to iterate over * @return an iterator */ public static EWAHIterator getEWAHIterator(EWAHCompressedBitmap bitmap) { @@ -45,7 +45,7 @@ * @return the long[] */ public long[] buffer() { - return this.rlw.array; + return this.rlw.parent.buffer; } /** @@ -77,6 +77,14 @@ return this.rlw; } + @Override + public EWAHIterator clone() throws CloneNotSupportedException { + EWAHIterator ans = (EWAHIterator) super.clone(); + ans.rlw = this.rlw.clone(); + ans.size = this.size; + ans.pointer = this.pointer; + return ans; + } /** The pointer represent the location of the current running length * word in the array of words (embedded in the rlw attribute). */ int pointer; diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/FastAggregation.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/FastAggregation.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/FastAggregation.java 2013-06-11 17:48:37.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/FastAggregation.java 2013-11-12 14:31:20.000000000 +0000 @@ -1,74 +1,436 @@ package com.googlecode.javaewah; +import java.util.Arrays; import java.util.Comparator; import java.util.PriorityQueue; + +/* + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. + */ + /** - * Fast algorithms to aggregate many bitmaps. - * These algorithms are just given as reference. - * They may not be faster than the corresponding - * methods in the EWAHCompressedBitmap class. + * Fast algorithms to aggregate many bitmaps. These algorithms are just given as + * reference. They may not be faster than the corresponding methods in the + * EWAHCompressedBitmap class. * * @author Daniel Lemire - * + * */ public class FastAggregation { + /** + * Compute the and aggregate using a temporary uncompressed bitmap. + * @param bitmaps the source bitmaps + * @param bufsize buffer size used during the computation in 64-bit words (per input bitmap) + * @return the or aggregate. + */ + public static EWAHCompressedBitmap bufferedand(final int bufsize, + final EWAHCompressedBitmap... bitmaps) { + EWAHCompressedBitmap answer = new EWAHCompressedBitmap(); + bufferedandWithContainer(answer,bufsize, bitmaps); + return answer; + } + /** + * Compute the and aggregate using a temporary uncompressed bitmap. + * + * @param container where the aggregate is written + * @param bufsize buffer size used during the computation in 64-bit words (per input bitmap) + * @param bitmaps the source bitmaps + */ + public static void bufferedandWithContainer(final BitmapStorage container,final int bufsize, + final EWAHCompressedBitmap... bitmaps) { + + java.util.LinkedList al = new java.util.LinkedList(); + for (EWAHCompressedBitmap bitmap : bitmaps) { + al.add(new IteratingBufferedRunningLengthWord(bitmap)); + } + + long[] hardbitmap = new long[bufsize*bitmaps.length]; + + for(IteratingRLW i : al) + if (i.size() == 0) { + al.clear(); + break; + } + + while (!al.isEmpty()) { + Arrays.fill(hardbitmap, ~0l); + long effective = Integer.MAX_VALUE; + for(IteratingRLW i : al) { + int eff = IteratorAggregation.inplaceand(hardbitmap, i); + if (eff < effective) + effective = eff; + } + for (int k = 0; k < effective; ++k) + container.add(hardbitmap[k]); + for(IteratingRLW i : al) + if (i.size() == 0) { + al.clear(); + break; + } + } + } + + /** + * Compute the or aggregate using a temporary uncompressed bitmap. + * @param bitmaps the source bitmaps + * @param bufsize buffer size used during the computation in 64-bit words + * @return the or aggregate. + */ + public static EWAHCompressedBitmap bufferedor(final int bufsize, + final EWAHCompressedBitmap... bitmaps) { + EWAHCompressedBitmap answer = new EWAHCompressedBitmap(); + bufferedorWithContainer(answer, bufsize, bitmaps); + return answer; + } + + /** + * Compute the or aggregate using a temporary uncompressed bitmap. + * + * @param container where the aggregate is written + * @param bufsize buffer size used during the computation in 64-bit words + * @param bitmaps the source bitmaps + */ + public static void bufferedorWithContainer(final BitmapStorage container, final int bufsize, + final EWAHCompressedBitmap... bitmaps) { + int range = 0; + EWAHCompressedBitmap[] sbitmaps = bitmaps.clone(); + Arrays.sort(sbitmaps, new Comparator() { + @Override + public int compare(EWAHCompressedBitmap a, EWAHCompressedBitmap b) { + return b.sizeinbits - a.sizeinbits; + } + }); + + java.util.ArrayList al = new java.util.ArrayList(); + for (EWAHCompressedBitmap bitmap : sbitmaps) { + if (bitmap.sizeinbits > range) + range = bitmap.sizeinbits; + al.add(new IteratingBufferedRunningLengthWord(bitmap)); + } + long[] hardbitmap = new long[bufsize]; + int maxr = al.size(); + while (maxr > 0) { + long effective = 0; + for (int k = 0; k < maxr; ++k) { + if (al.get(k).size() > 0) { + int eff = IteratorAggregation.inplaceor(hardbitmap, al.get(k)); + if (eff > effective) + effective = eff; + } else + maxr = k; + } + for (int k = 0; k < effective; ++k) + container.add(hardbitmap[k]); + Arrays.fill(hardbitmap, 0); + + } + container.setSizeInBits(range); + } + + /** + * Compute the xor aggregate using a temporary uncompressed bitmap. + * @param bitmaps the source bitmaps + * @param bufsize buffer size used during the computation in 64-bit words + * @return the xor aggregate. + */ + public static EWAHCompressedBitmap bufferedxor(final int bufsize, + final EWAHCompressedBitmap... bitmaps) { + EWAHCompressedBitmap answer = new EWAHCompressedBitmap(); + bufferedxorWithContainer(answer, bufsize,bitmaps); + return answer; + } + + + /** + * Compute the xor aggregate using a temporary uncompressed bitmap. + * + * @param container where the aggregate is written + * @param bufsize buffer size used during the computation in 64-bit words + * @param bitmaps the source bitmaps + */ + public static void bufferedxorWithContainer(final BitmapStorage container, final int bufsize, + final EWAHCompressedBitmap... bitmaps) { + int range = 0; + EWAHCompressedBitmap[] sbitmaps = bitmaps.clone(); + Arrays.sort(sbitmaps, new Comparator() { + @Override + public int compare(EWAHCompressedBitmap a, EWAHCompressedBitmap b) { + return b.sizeinbits - a.sizeinbits; + } + }); + + java.util.ArrayList al = new java.util.ArrayList(); + for (EWAHCompressedBitmap bitmap : sbitmaps) { + if (bitmap.sizeinbits > range) + range = bitmap.sizeinbits; + al.add(new IteratingBufferedRunningLengthWord(bitmap)); + } + long[] hardbitmap = new long[bufsize]; + int maxr = al.size(); + while (maxr > 0) { + long effective = 0; + for (int k = 0; k < maxr; ++k) { + if (al.get(k).size() > 0) { + int eff = IteratorAggregation.inplacexor(hardbitmap, al.get(k)); + if (eff > effective) + effective = eff; + } else + maxr = k; + } + for (int k = 0; k < effective; ++k) + container.add(hardbitmap[k]); + Arrays.fill(hardbitmap, 0); + } + container.setSizeInBits(range); + } + + /** + * Uses a priority queue to compute the or aggregate. + * @param a class extending LogicalElement (like a compressed bitmap) + * @param bitmaps + * bitmaps to be aggregated + * @return the or aggregate + */ + @SuppressWarnings({ "rawtypes", "unchecked" }) + public static T or(T... bitmaps) { + PriorityQueue pq = new PriorityQueue(bitmaps.length, + new Comparator() { + @Override + public int compare(T a, T b) { + return a.sizeInBytes() - b.sizeInBytes(); + } + }); + for (T x : bitmaps) { + pq.add(x); + } + while (pq.size() > 1) { + T x1 = pq.poll(); + T x2 = pq.poll(); + pq.add((T) x1.or(x2)); + } + return pq.poll(); + } + /** + * Uses a priority queue to compute the or aggregate. + * @param container where we write the result + * @param bitmaps to be aggregated + */ + public static void orToContainer(final BitmapStorage container, + final EWAHCompressedBitmap ... bitmaps) { + if(bitmaps.length < 2) throw new IllegalArgumentException("We need at least two bitmaps"); + PriorityQueue pq = new PriorityQueue(bitmaps.length, + new Comparator() { + @Override + public int compare(EWAHCompressedBitmap a, EWAHCompressedBitmap b) { + return a.sizeInBytes() - b.sizeInBytes(); + } + }); + for (EWAHCompressedBitmap x : bitmaps) { + pq.add(x); + } + while (pq.size() > 2) { + EWAHCompressedBitmap x1 = pq.poll(); + EWAHCompressedBitmap x2 = pq.poll(); + pq.add(x1.or(x2)); + } + pq.poll().orToContainer(pq.poll(), container); + } + + + /** + * Uses a priority queue to compute the xor aggregate. + * + * @param a class extending LogicalElement (like a compressed bitmap) + * @param bitmaps + * bitmaps to be aggregated + * @return the xor aggregate + */ + @SuppressWarnings({ "rawtypes", "unchecked" }) + public static T xor(T... bitmaps) { + PriorityQueue pq = new PriorityQueue(bitmaps.length, + new Comparator() { + + @Override + public int compare(T a, T b) { + return a.sizeInBytes() - b.sizeInBytes(); + } + }); + for (T x : bitmaps) + pq.add(x); + while (pq.size() > 1) { + T x1 = pq.poll(); + T x2 = pq.poll(); + pq.add((T) x1.xor(x2)); + } + return pq.poll(); + } + + /** + * Uses a priority queue to compute the xor aggregate. + * @param container where we write the result + * @param bitmaps to be aggregated + */ + public static void xorToContainer(final BitmapStorage container, + final EWAHCompressedBitmap ... bitmaps) { + if(bitmaps.length < 2) throw new IllegalArgumentException("We need at least two bitmaps"); + PriorityQueue pq = new PriorityQueue(bitmaps.length, + new Comparator() { + @Override + public int compare(EWAHCompressedBitmap a, EWAHCompressedBitmap b) { + return a.sizeInBytes() - b.sizeInBytes(); + } + }); + for (EWAHCompressedBitmap x : bitmaps) { + pq.add(x); + } + while (pq.size() > 2) { + EWAHCompressedBitmap x1 = pq.poll(); + EWAHCompressedBitmap x2 = pq.poll(); + pq.add(x1.xor(x2)); + } + pq.poll().xorToContainer(pq.poll(), container); + } + + /** + * For internal use. Computes the bitwise or of the provided bitmaps and + * stores the result in the container. (This used to be the default.) + * + * @deprecated use EWAHCompressedBitmap.or instead + * @since 0.4.0 + * @param container where store the result + * @param bitmaps to be aggregated + */ + @Deprecated + public static void legacy_orWithContainer(final BitmapStorage container, + final EWAHCompressedBitmap... bitmaps) { + if (bitmaps.length == 2) { + // should be more efficient + bitmaps[0].orToContainer(bitmaps[1], container); + return; + } + + // Sort the bitmaps in descending order by sizeinbits. We will exhaust the + // sorted bitmaps from right to left. + final EWAHCompressedBitmap[] sortedBitmaps = bitmaps.clone(); + Arrays.sort(sortedBitmaps, new Comparator() { + @Override + public int compare(EWAHCompressedBitmap a, EWAHCompressedBitmap b) { + return a.sizeinbits < b.sizeinbits ? 1 + : a.sizeinbits == b.sizeinbits ? 0 : -1; + } + }); + + final IteratingBufferedRunningLengthWord[] rlws = new IteratingBufferedRunningLengthWord[bitmaps.length]; + int maxAvailablePos = 0; + for (EWAHCompressedBitmap bitmap : sortedBitmaps) { + EWAHIterator iterator = bitmap.getEWAHIterator(); + if (iterator.hasNext()) { + rlws[maxAvailablePos++] = new IteratingBufferedRunningLengthWord( + iterator); + } + } + + if (maxAvailablePos == 0) { // this never happens... + container.setSizeInBits(0); + return; + } + + int maxSize = sortedBitmaps[0].sizeinbits; + + while (true) { + long maxOneRl = 0; + long minZeroRl = Long.MAX_VALUE; + long minSize = Long.MAX_VALUE; + int numEmptyRl = 0; + for (int i = 0; i < maxAvailablePos; i++) { + IteratingBufferedRunningLengthWord rlw = rlws[i]; + long size = rlw.size(); + if (size == 0) { + maxAvailablePos = i; + break; + } + minSize = Math.min(minSize, size); + + if (rlw.getRunningBit()) { + long rl = rlw.getRunningLength(); + maxOneRl = Math.max(maxOneRl, rl); + minZeroRl = 0; + if (rl == 0 && size > 0) { + numEmptyRl++; + } + } else { + long rl = rlw.getRunningLength(); + minZeroRl = Math.min(minZeroRl, rl); + if (rl == 0 && size > 0) { + numEmptyRl++; + } + } + } + + if (maxAvailablePos == 0) { + break; + } else if (maxAvailablePos == 1) { + // only one bitmap is left so just write the rest of it out + rlws[0].discharge(container); + break; + } + + if (maxOneRl > 0) { + container.addStreamOfEmptyWords(true, maxOneRl); + for (int i = 0; i < maxAvailablePos; i++) { + IteratingBufferedRunningLengthWord rlw = rlws[i]; + rlw.discardFirstWords(maxOneRl); + } + } else if (minZeroRl > 0) { + container.addStreamOfEmptyWords(false, minZeroRl); + for (int i = 0; i < maxAvailablePos; i++) { + IteratingBufferedRunningLengthWord rlw = rlws[i]; + rlw.discardFirstWords(minZeroRl); + } + } else { + int index = 0; + + if (numEmptyRl == 1) { + // if one rlw has literal words to process and the rest have a run of + // 0's we can write them out here + IteratingBufferedRunningLengthWord emptyRl = null; + long minNonEmptyRl = Long.MAX_VALUE; + for (int i = 0; i < maxAvailablePos; i++) { + IteratingBufferedRunningLengthWord rlw = rlws[i]; + long rl = rlw.getRunningLength(); + if (rl == 0) { + assert emptyRl == null; + emptyRl = rlw; + } else { + minNonEmptyRl = Math.min(minNonEmptyRl, rl); + } + } + long wordsToWrite = minNonEmptyRl > minSize ? minSize : minNonEmptyRl; + if (emptyRl != null) + emptyRl.writeLiteralWords((int) wordsToWrite, container); + index += wordsToWrite; + } - @SuppressWarnings({ "rawtypes", "unchecked" }) - public static T and(T...bitmaps) { - // for "and" a priority queue is not needed, but - // overhead ought to be small - PriorityQueue pq = new PriorityQueue( - bitmaps.length, new Comparator() { - public int compare(T a, T b) { - return a.sizeInBytes() - b.sizeInBytes(); - } - }); - for (T x : bitmaps) - pq.add(x); - while (pq.size() > 1) { - T x1 = pq.poll(); - T x2 = pq.poll(); - pq.add( (T) x1.and(x2)); - } - return pq.poll(); - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - public static T or(T...bitmaps) { - PriorityQueue pq = new PriorityQueue( - bitmaps.length, new Comparator() { - public int compare(T a, T b) { - return a.sizeInBytes() - b.sizeInBytes(); - } - }); - for (T x : bitmaps) { - pq.add(x); - } - while (pq.size() > 1) { - T x1 = pq.poll(); - T x2 = pq.poll(); - pq.add( (T) x1.or(x2)); - } - return pq.poll(); - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - public static T xor(T...bitmaps) { - PriorityQueue pq = new PriorityQueue( - bitmaps.length, new Comparator() { - - public int compare(T a, T b) { - return a.sizeInBytes() - b.sizeInBytes(); - } - }); - for (T x : bitmaps) - pq.add(x); - while (pq.size() > 1) { - T x1 = pq.poll(); - T x2 = pq.poll(); - pq.add( (T) x1.xor(x2)); - } - return pq.poll(); - } + while (index < minSize) { + long word = 0; + for (int i = 0; i < maxAvailablePos; i++) { + IteratingBufferedRunningLengthWord rlw = rlws[i]; + if (rlw.getRunningLength() <= index) { + word |= rlw.getLiteralWordAt(index - (int) rlw.getRunningLength()); + } + } + container.add(word); + index++; + } + for (int i = 0; i < maxAvailablePos; i++) { + IteratingBufferedRunningLengthWord rlw = rlws[i]; + rlw.discardFirstWords(minSize); + } + } + } + container.setSizeInBits(maxSize); + } + } diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/IntIteratorImpl.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/IntIteratorImpl.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/IntIteratorImpl.java 2013-06-11 17:48:37.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/IntIteratorImpl.java 2013-11-12 14:31:20.000000000 +0000 @@ -2,7 +2,7 @@ /* * Copyright 2012, Google Inc. - * Licensed under APL 2.0. + * Licensed under the Apache License, Version 2.0. */ import static com.googlecode.javaewah.EWAHCompressedBitmap.wordinbits; @@ -43,11 +43,13 @@ return true; } - public boolean hasNext() { + @Override +public boolean hasNext() { return this.hasnext; } - public final int next() { + @Override +public final int next() { final int answer; if (runningHasNext()) { answer = this.position++; diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/IntIterator.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/IntIterator.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/IntIterator.java 2013-06-11 17:48:37.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/IntIterator.java 2013-11-12 14:31:20.000000000 +0000 @@ -1,8 +1,8 @@ package com.googlecode.javaewah; /* - * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc. and Veronika Zenz - * Licensed under APL 2.0. + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. */ /** diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/IntIteratorOverIteratingRLW.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/IntIteratorOverIteratingRLW.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/IntIteratorOverIteratingRLW.java 1970-01-01 00:00:00.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/IntIteratorOverIteratingRLW.java 2013-11-12 14:31:20.000000000 +0000 @@ -0,0 +1,89 @@ +package com.googlecode.javaewah; + +import static com.googlecode.javaewah.EWAHCompressedBitmap.wordinbits; + + +/* + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. + */ +/** + * Implementation of an IntIterator over an IteratingRLW. + * + * + */ +public class IntIteratorOverIteratingRLW implements IntIterator { + IteratingRLW parent; + private int position; + private int runningLength; + private long word; + private int wordPosition; + private int wordLength; + private int literalPosition; + private boolean hasnext; + + /** + * @param p iterator we wish to iterate over + */ + public IntIteratorOverIteratingRLW(final IteratingRLW p) { + this.parent = p; + this.position = 0; + setupForCurrentRunningLengthWord(); + this.hasnext = moveToNext(); + } + + /** + * @return whether we could find another set bit; don't move if there is an unprocessed value + */ + private final boolean moveToNext() { + while (!runningHasNext() && !literalHasNext()) { + if (this.parent.next()) + setupForCurrentRunningLengthWord(); + else return false; + } + return true; + } + + @Override + public boolean hasNext() { + return this.hasnext; + } + + @Override + public final int next() { + final int answer; + if (runningHasNext()) { + answer = this.position++; + } else { + final int bit = Long.numberOfTrailingZeros(this.word); + this.word ^= (1l << bit); + answer = this.literalPosition + bit; + } + this.hasnext = this.moveToNext(); + return answer; + } + + private final void setupForCurrentRunningLengthWord() { + this.runningLength = wordinbits * (int) this.parent.getRunningLength() + + this.position; + + if (!this.parent.getRunningBit()) { + this.position = this.runningLength; + } + this.wordPosition = 0; + this.wordLength = this.parent.getNumberOfLiteralWords(); + } + + private final boolean runningHasNext() { + return this.position < this.runningLength; + } + + private final boolean literalHasNext() { + while (this.word == 0 && this.wordPosition < this.wordLength) { + this.word = this.parent.getLiteralWordAt(this.wordPosition++); + this.literalPosition = this.position; + this.position += wordinbits; + } + return this.word != 0; + } +} diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/IteratingBufferedRunningLengthWord.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/IteratingBufferedRunningLengthWord.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/IteratingBufferedRunningLengthWord.java 2013-06-11 17:48:37.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/IteratingBufferedRunningLengthWord.java 2013-11-12 14:31:20.000000000 +0000 @@ -1,8 +1,8 @@ package com.googlecode.javaewah; /* - * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc. and Veronika Zenz - * Licensed under APL 2.0. + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. */ /** * Mostly for internal use. Similar to BufferedRunningLengthWord, but automatically @@ -11,7 +11,7 @@ * @since 0.4.0 * @author David McIntosh */ -public final class IteratingBufferedRunningLengthWord { +public final class IteratingBufferedRunningLengthWord implements IteratingRLW, Cloneable{ /** * Instantiates a new iterating buffered running length word. * @@ -24,21 +24,30 @@ this.buffer = this.iterator.buffer(); } - /** - * Instantiates a new iterating buffered running length word. - * - * @param iterator iterator - */ - public IteratingBufferedRunningLengthWord(final EWAHCompressedBitmap bitmap) { - this(EWAHIterator.getEWAHIterator(bitmap)); - } + + + /** + * Instantiates a new iterating buffered running length word. + * @param bitmap over which we want to iterate + * + */ +public IteratingBufferedRunningLengthWord(final EWAHCompressedBitmap bitmap) { + this.iterator = EWAHIterator.getEWAHIterator(bitmap); + this.brlw = new BufferedRunningLengthWord(this.iterator.next()); + this.literalWordStartPosition = this.iterator.literalWords() + this.brlw.literalwordoffset; + this.buffer = this.iterator.buffer(); + } + + + /** * Discard first words, iterating to the next running length word if needed. * - * @param x the x + * @param x the number of words to be discarded */ - public void discardFirstWords(long x) { + @Override +public void discardFirstWords(long x) { while (x > 0) { if (this.brlw.RunningLength > x) { this.brlw.RunningLength -= x; @@ -60,6 +69,21 @@ } } } + /** + * Move to the next RunningLengthWord + * @return whether the move was possible + */ + @Override +public boolean next() { + if (!this.iterator.hasNext()) { + this.brlw.NumberOfLiteralWords = 0; + this.brlw.RunningLength = 0; + return false; + } + this.brlw.reset(this.iterator.next()); + this.literalWordStartPosition = this.iterator.literalWords(); // + this.brlw.literalwordoffset ==0 + return true; + } /** * Write out up to max words, returns how many were written @@ -103,14 +127,13 @@ pl = max - index; } container.addStreamOfEmptyWords(!getRunningBit(), pl); - discardFirstWords(pl); index += pl; int pd = getNumberOfLiteralWords(); if (pd + index > max) { pd = (int) (max - index); } writeNegatedLiteralWords(pd, container); - discardFirstWords(pd); + discardFirstWords(pl+pd); index += pd; } return index; @@ -144,7 +167,8 @@ * @param index zero based index * @return the literal word */ - public long getLiteralWordAt(int index) { + @Override +public long getLiteralWordAt(int index) { return this.buffer[this.literalWordStartPosition + index]; } @@ -153,7 +177,8 @@ * * @return the number of literal words */ - public int getNumberOfLiteralWords() { + @Override +public int getNumberOfLiteralWords() { return this.brlw.NumberOfLiteralWords; } @@ -162,7 +187,8 @@ * * @return the running bit */ - public boolean getRunningBit() { + @Override +public boolean getRunningBit() { return this.brlw.RunningBit; } @@ -171,7 +197,8 @@ * * @return the running length */ - public long getRunningLength() { + @Override +public long getRunningLength() { return this.brlw.RunningLength; } @@ -180,14 +207,15 @@ * * @return the long */ - public long size() { + @Override +public long size() { return this.brlw.size(); } /** * write the first N literal words to the target bitmap. Does not discard the words or perform iteration. - * @param numWords - * @param container + * @param numWords number of words to be written + * @param container where we write */ public void writeLiteralWords(int numWords, BitmapStorage container) { container.addStreamOfLiteralWords(this.buffer, this.literalWordStartPosition, numWords); @@ -195,15 +223,15 @@ /** * write the first N literal words (negated) to the target bitmap. Does not discard the words or perform iteration. - * @param numWords - * @param container + * @param numWords number of words to be written + * @param container where we write */ public void writeNegatedLiteralWords(int numWords, BitmapStorage container) { container.addStreamOfNegatedLiteralWords(this.buffer, this.literalWordStartPosition, numWords); } /** - * For internal use. (One could use the non-static dischard method instead, + * For internal use. (One could use the non-static discharge method instead, * but we expect them to be slower.) * * @param initialWord @@ -228,6 +256,18 @@ runningLengthWord = new BufferedRunningLengthWord(iterator.next()); } } + + + @Override + public IteratingBufferedRunningLengthWord clone() throws CloneNotSupportedException { + IteratingBufferedRunningLengthWord answer = (IteratingBufferedRunningLengthWord) super.clone(); + answer.brlw = this.brlw.clone(); + answer.buffer = this.buffer; + answer.iterator = this.iterator.clone(); + answer.literalWordStartPosition = this.literalWordStartPosition; + return answer; + } + private BufferedRunningLengthWord brlw; private long[] buffer; diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/IteratingRLW.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/IteratingRLW.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/IteratingRLW.java 1970-01-01 00:00:00.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/IteratingRLW.java 2013-11-12 14:31:20.000000000 +0000 @@ -0,0 +1,49 @@ +package com.googlecode.javaewah; + + +/* + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. + */ + +/** + * High-level iterator over a compressed bitmap. + * + */ +public interface IteratingRLW { + /** + * @return whether there is more + */ + public boolean next() ; + /** + * @param index where the literal word is + * @return the literal word at the given index. + */ + public long getLiteralWordAt(int index); + /** + * @return the number of literal (non-fill) words + */ + public int getNumberOfLiteralWords() ; + /** + * @return the bit used for the fill bits + */ + public boolean getRunningBit() ; + /** + * @return sum of getRunningLength() and getNumberOfLiteralWords() + */ + public long size() ; + /** + * @return length of the run of fill words + */ + public long getRunningLength() ; + /** + * @param x the number of words to discard + */ + public void discardFirstWords(long x); + + /** + * @return a copy of the iterator + * @throws CloneNotSupportedException this should not be thrown in theory + */ + public IteratingRLW clone() throws CloneNotSupportedException; +} diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/IteratorAggregation.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/IteratorAggregation.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/IteratorAggregation.java 1970-01-01 00:00:00.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/IteratorAggregation.java 2013-11-12 14:31:20.000000000 +0000 @@ -0,0 +1,616 @@ +package com.googlecode.javaewah; + +import java.util.Arrays; +import java.util.Iterator; +import java.util.LinkedList; + +/* + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. + */ + +/** + * Set of helper functions to aggregate bitmaps. + * + */ +public class IteratorAggregation { + + /** + * @param x iterator to negate + * @return negated version of the iterator + */ + public static IteratingRLW not(final IteratingRLW x) { + return new IteratingRLW() { + + @Override + public boolean next() { + return x.next(); + } + + @Override + public long getLiteralWordAt(int index) { + return ~x.getLiteralWordAt(index); + } + + @Override + public int getNumberOfLiteralWords() { + return x.getNumberOfLiteralWords(); + } + + @Override + public boolean getRunningBit() { + return ! x.getRunningBit(); + } + + @Override + public long size() { + return x.size(); + } + + @Override + public long getRunningLength() { + return x.getRunningLength(); + } + + @Override + public void discardFirstWords(long y) { + x.discardFirstWords(y); + } + + @Override + public IteratingRLW clone() throws CloneNotSupportedException { + throw new CloneNotSupportedException(); + } + + + }; + } + + /** + * Aggregate the iterators using a bitmap buffer. + * + * @param al set of iterators to aggregate + * @return and aggregate + */ + public static IteratingRLW bufferedand(final IteratingRLW... al) { + return bufferedand(DEFAULTMAXBUFSIZE,al); + } + + /** + * Aggregate the iterators using a bitmap buffer. + * + * @param al set of iterators to aggregate + * @param bufsize size of the internal buffer used by the iterator in 64-bit words (per input iterator) + * @return and aggregate + */ + public static IteratingRLW bufferedand(final int bufsize, final IteratingRLW... al) { + if (al.length == 0) + throw new IllegalArgumentException("Need at least one iterator"); + if (al.length == 1) + return al[0]; + final LinkedList basell = new LinkedList(); + for (IteratingRLW i : al) + basell.add(i); + return new BufferedIterator(new BufferedAndIterator(basell,bufsize)); + } + + /** + * Aggregate the iterators using a bitmap buffer. + * + * @param al set of iterators to aggregate + * @return or aggregate + */ + public static IteratingRLW bufferedor(final IteratingRLW... al) { + return bufferedor(DEFAULTMAXBUFSIZE,al); + } + + + /** + * Aggregate the iterators using a bitmap buffer. + * + * @param al iterators to aggregate + * @param bufsize size of the internal buffer used by the iterator in 64-bit words + * @return or aggregate + */ + public static IteratingRLW bufferedor(final int bufsize, final IteratingRLW... al) { + if (al.length == 0) + throw new IllegalArgumentException("Need at least one iterator"); + if (al.length == 1) + return al[0]; + + final LinkedList basell = new LinkedList(); + for (IteratingRLW i : al) + basell.add(i); + return new BufferedIterator(new BufferedORIterator(basell,bufsize)); + } + + /** + * Aggregate the iterators using a bitmap buffer. + * + * @param al set of iterators to aggregate + * @return xor aggregate + */ + public static IteratingRLW bufferedxor(final IteratingRLW... al) { + return bufferedxor(DEFAULTMAXBUFSIZE,al); + } + + + /** + * Aggregate the iterators using a bitmap buffer. + * + * @param al iterators to aggregate + * @param bufsize size of the internal buffer used by the iterator in 64-bit words + * @return xor aggregate + */ + public static IteratingRLW bufferedxor(final int bufsize, final IteratingRLW... al) { + if (al.length == 0) + throw new IllegalArgumentException("Need at least one iterator"); + if (al.length == 1) + return al[0]; + + + final LinkedList basell = new LinkedList(); + for (IteratingRLW i : al) + basell.add(i); + + return new BufferedIterator(new BufferedXORIterator(basell, bufsize)); + } + + + /** + * Write out the content of the iterator, but as if it were all zeros. + * + * @param container + * where we write + * @param i + * the iterator + */ + protected static void dischargeAsEmpty(final BitmapStorage container, + final IteratingRLW i) { + while (i.size() > 0) { + container.addStreamOfEmptyWords(false, i.size()); + i.next(); + + } + } + + /** + * Write out up to max words, returns how many were written + * @param container target for writes + * @param i source of data + * @param max maximal number of writes + * @return how many written + */ + + protected static long discharge(final BitmapStorage container, IteratingRLW i, long max) { + long counter = 0; + while (i.size() > 0 && counter < max) { + long L1 = i.getRunningLength(); + if (L1 > 0) { + if (L1 + counter > max) + L1 = max - counter; + container.addStreamOfEmptyWords(i.getRunningBit(), L1); + counter += L1; + } + long L = i.getNumberOfLiteralWords(); + if(L + counter > max) L = max - counter; + for (int k = 0; k < L; ++k) { + container.add(i.getLiteralWordAt(k)); + } + counter += L; + i.discardFirstWords(L+L1); + } + return counter; + } + + + /** + * Write out up to max negated words, returns how many were written + * @param container target for writes + * @param i source of data + * @param max maximal number of writes + * @return how many written + */ + protected static long dischargeNegated(final BitmapStorage container, IteratingRLW i, long max) { + long counter = 0; + while (i.size() > 0 && counter < max) { + long L1 = i.getRunningLength(); + if (L1 > 0) { + if (L1 + counter > max) + L1 = max - counter; + container.addStreamOfEmptyWords(!i.getRunningBit(), L1); + counter += L1; + } + long L = i.getNumberOfLiteralWords(); + if(L + counter > max) L = max - counter; + for (int k = 0; k < L; ++k) { + container.add(~i.getLiteralWordAt(k)); + } + counter += L; + i.discardFirstWords(L+L1); + } + return counter; + } + + static void andToContainer(final BitmapStorage container, + int desiredrlwcount, final IteratingRLW rlwi, IteratingRLW rlwj) { + while ((rlwi.size()>0) && (rlwj.size()>0) && (desiredrlwcount-- >0) ) { + while ((rlwi.getRunningLength() > 0) || (rlwj.getRunningLength() > 0)) { + final boolean i_is_prey = rlwi.getRunningLength() < rlwj + .getRunningLength(); + final IteratingRLW prey = i_is_prey ? rlwi : rlwj; + final IteratingRLW predator = i_is_prey ? rlwj + : rlwi; + if (predator.getRunningBit() == false) { + container.addStreamOfEmptyWords(false, predator.getRunningLength()); + prey.discardFirstWords(predator.getRunningLength()); + predator.discardFirstWords(predator.getRunningLength()); + } else { + final long index = discharge(container, prey, predator.getRunningLength()); + container.addStreamOfEmptyWords(false, predator.getRunningLength() + - index); + predator.discardFirstWords(predator.getRunningLength()); + } + } + final int nbre_literal = Math.min(rlwi.getNumberOfLiteralWords(), + rlwj.getNumberOfLiteralWords()); + if (nbre_literal > 0) { + desiredrlwcount -= nbre_literal; + for (int k = 0; k < nbre_literal; ++k) + container.add(rlwi.getLiteralWordAt(k) & rlwj.getLiteralWordAt(k)); + rlwi.discardFirstWords(nbre_literal); + rlwj.discardFirstWords(nbre_literal); + } + } + } + + static void andToContainer(final BitmapStorage container, + final IteratingRLW rlwi, IteratingRLW rlwj) { + while ((rlwi.size()>0) && (rlwj.size()>0) ) { + while ((rlwi.getRunningLength() > 0) || (rlwj.getRunningLength() > 0)) { + final boolean i_is_prey = rlwi.getRunningLength() < rlwj + .getRunningLength(); + final IteratingRLW prey = i_is_prey ? rlwi : rlwj; + final IteratingRLW predator = i_is_prey ? rlwj + : rlwi; + if (predator.getRunningBit() == false) { + container.addStreamOfEmptyWords(false, predator.getRunningLength()); + prey.discardFirstWords(predator.getRunningLength()); + predator.discardFirstWords(predator.getRunningLength()); + } else { + final long index = discharge(container, prey, predator.getRunningLength()); + container.addStreamOfEmptyWords(false, predator.getRunningLength() + - index); + predator.discardFirstWords(predator.getRunningLength()); + } + } + final int nbre_literal = Math.min(rlwi.getNumberOfLiteralWords(), + rlwj.getNumberOfLiteralWords()); + if (nbre_literal > 0) { + for (int k = 0; k < nbre_literal; ++k) + container.add(rlwi.getLiteralWordAt(k) & rlwj.getLiteralWordAt(k)); + rlwi.discardFirstWords(nbre_literal); + rlwj.discardFirstWords(nbre_literal); + } + } + } + + + /** + * Compute the first few words of the XOR aggregate between two iterators. + * + * @param container where to write + * @param desiredrlwcount number of words to be written (max) + * @param rlwi first iterator to aggregate + * @param rlwj second iterator to aggregate + */ + public static void xorToContainer(final BitmapStorage container, + int desiredrlwcount, final IteratingRLW rlwi, final IteratingRLW rlwj) { + while ((rlwi.size()>0) && (rlwj.size()>0) && (desiredrlwcount-- >0) ) { + while ((rlwi.getRunningLength() > 0) || (rlwj.getRunningLength() > 0)) { + final boolean i_is_prey = rlwi.getRunningLength() < rlwj + .getRunningLength(); + final IteratingRLW prey = i_is_prey ? rlwi : rlwj; + final IteratingRLW predator = i_is_prey ? rlwj + : rlwi; + if (predator.getRunningBit() == false) { + long index = discharge(container, prey, predator.getRunningLength()); + container.addStreamOfEmptyWords(false, predator.getRunningLength() + - index); + predator.discardFirstWords(predator.getRunningLength()); + } else { + long index = dischargeNegated(container, prey, predator.getRunningLength()); + container.addStreamOfEmptyWords(true, predator.getRunningLength() + - index); + predator.discardFirstWords(predator.getRunningLength()); + } + } + final int nbre_literal = Math.min(rlwi.getNumberOfLiteralWords(), + rlwj.getNumberOfLiteralWords()); + if (nbre_literal > 0) { + desiredrlwcount -= nbre_literal; + for (int k = 0; k < nbre_literal; ++k) + container.add(rlwi.getLiteralWordAt(k) ^ rlwj.getLiteralWordAt(k)); + rlwi.discardFirstWords(nbre_literal); + rlwj.discardFirstWords(nbre_literal); + } + } + } + + protected static int inplaceor(long[] bitmap, + IteratingRLW i) { + + int pos = 0; + long s; + while ((s = i.size()) > 0) { + if (pos + s < bitmap.length) { + final int L = (int) i.getRunningLength(); + if (i.getRunningBit()) + java.util.Arrays.fill(bitmap, pos, pos + L, ~0l); + pos += L; + final int LR = i.getNumberOfLiteralWords(); + + for (int k = 0; k < LR; ++k) + bitmap[pos++] |= i.getLiteralWordAt(k); + if (!i.next()) { + return pos; + } + } else { + int howmany = bitmap.length - pos; + int L = (int) i.getRunningLength(); + + if (pos + L > bitmap.length) { + if (i.getRunningBit()) { + java.util.Arrays.fill(bitmap, pos, bitmap.length, ~0l); + } + i.discardFirstWords(howmany); + return bitmap.length; + } + if (i.getRunningBit()) + java.util.Arrays.fill(bitmap, pos, pos + L, ~0l); + pos += L; + for (int k = 0; pos < bitmap.length; ++k) + bitmap[pos++] |= i.getLiteralWordAt(k); + i.discardFirstWords(howmany); + return pos; + } + } + return pos; + } + + protected static int inplacexor(long[] bitmap, + IteratingRLW i) { + int pos = 0; + long s; + while ((s = i.size()) > 0) { + if (pos + s < bitmap.length) { + final int L = (int) i.getRunningLength(); + if (i.getRunningBit()) { + for(int k = pos ; k < pos + L; ++k) + bitmap[k] = ~bitmap[k]; + } + pos += L; + final int LR = i.getNumberOfLiteralWords(); + for (int k = 0; k < LR; ++k) + bitmap[pos++] ^= i.getLiteralWordAt(k); + if (!i.next()) { + return pos; + } + } else { + int howmany = bitmap.length - pos; + int L = (int) i.getRunningLength(); + if (pos + L > bitmap.length) { + if (i.getRunningBit()) { + for(int k = pos ; k < bitmap.length; ++k) + bitmap[k] = ~bitmap[k]; + } + i.discardFirstWords(howmany); + return bitmap.length; + } + if (i.getRunningBit()) + for(int k = pos ; k < pos + L; ++k) + bitmap[k] = ~bitmap[k]; + pos += L; + for (int k = 0; pos < bitmap.length; ++k) + bitmap[pos++] ^= i.getLiteralWordAt(k); + i.discardFirstWords(howmany); + return pos; + } + } + return pos; + } + protected static int inplaceand(long[] bitmap, + IteratingRLW i) { + int pos = 0; + long s; + while ((s = i.size()) > 0) { + if (pos + s < bitmap.length) { + final int L = (int) i.getRunningLength(); + if (!i.getRunningBit()) { + for(int k = pos ; k < pos + L; ++k) + bitmap[k] = 0; + } + pos += L; + final int LR = i.getNumberOfLiteralWords(); + for (int k = 0; k < LR; ++k) + bitmap[pos++] &= i.getLiteralWordAt(k); + if (!i.next()) { + return pos; + } + } else { + int howmany = bitmap.length - pos; + int L = (int) i.getRunningLength(); + if (pos + L > bitmap.length) { + if (!i.getRunningBit()) { + for(int k = pos ; k < bitmap.length; ++k) + bitmap[k] = 0; + } + i.discardFirstWords(howmany); + return bitmap.length; + } + if (!i.getRunningBit()) + for(int k = pos ; k < pos + L; ++k) + bitmap[k] = 0; + pos += L; + for (int k = 0; pos < bitmap.length; ++k) + bitmap[pos++] &= i.getLiteralWordAt(k); + i.discardFirstWords(howmany); + return pos; + } + } + return pos; + } + + /** + * An optimization option. Larger values may improve speed, but at + * the expense of memory. + */ + public final static int DEFAULTMAXBUFSIZE = 65536; +} +class BufferedORIterator implements CloneableIterator { + EWAHCompressedBitmap buffer = new EWAHCompressedBitmap(); + long[] hardbitmap; + LinkedList ll; + int buffersize; + + BufferedORIterator(LinkedList basell, int bufsize) { + this.ll = basell; + this.hardbitmap = new long[bufsize]; + } + + @Override + public BufferedXORIterator clone() throws CloneNotSupportedException { + BufferedXORIterator answer = (BufferedXORIterator) super.clone(); + answer.buffer = this.buffer.clone(); + answer.hardbitmap = this.hardbitmap.clone(); + answer.ll = (LinkedList) this.ll.clone(); + return answer; + } + + @Override + public boolean hasNext() { + return !this.ll.isEmpty(); + } + + @Override + public EWAHIterator next() { + this.buffer.clear(); + long effective = 0; + Iterator i = this.ll.iterator(); + while (i.hasNext()) { + IteratingRLW rlw = i.next(); + if (rlw.size() > 0) { + int eff = IteratorAggregation.inplaceor(this.hardbitmap, rlw); + if (eff > effective) + effective = eff; + } else + i.remove(); + } + for (int k = 0; k < effective; ++k) { + this.buffer.add(this.hardbitmap[k]); + } + + Arrays.fill(this.hardbitmap, 0); + return this.buffer.getEWAHIterator(); + } +} + +class BufferedXORIterator implements CloneableIterator { + EWAHCompressedBitmap buffer = new EWAHCompressedBitmap(); + long[] hardbitmap; + LinkedList ll; + int buffersize; + + BufferedXORIterator(LinkedList basell, int bufsize) { + this.ll = basell; + this.hardbitmap = new long[bufsize]; + } + + @Override + public BufferedXORIterator clone() throws CloneNotSupportedException { + BufferedXORIterator answer = (BufferedXORIterator) super.clone(); + answer.buffer = this.buffer.clone(); + answer.hardbitmap = this.hardbitmap.clone(); + answer.ll = (LinkedList) this.ll.clone(); + return answer; + } + + @Override + public boolean hasNext() { + return !this.ll.isEmpty(); + } + + @Override + public EWAHIterator next() { + this.buffer.clear(); + long effective = 0; + Iterator i = this.ll.iterator(); + while (i.hasNext()) { + IteratingRLW rlw = i.next(); + if (rlw.size() > 0) { + int eff = IteratorAggregation.inplacexor(this.hardbitmap, rlw); + if (eff > effective) + effective = eff; + } else + i.remove(); + } + for (int k = 0; k < effective; ++k) + this.buffer.add(this.hardbitmap[k]); + Arrays.fill(this.hardbitmap, 0); + return this.buffer.getEWAHIterator(); + } +} + + +class BufferedAndIterator implements CloneableIterator { + EWAHCompressedBitmap buffer = new EWAHCompressedBitmap(); + LinkedList ll; + int buffersize; + + public BufferedAndIterator(LinkedList basell, int bufsize) { + this.ll = basell; + this.buffersize = bufsize; + + } + + @Override + public boolean hasNext() { + return !this.ll.isEmpty(); + } + + @Override + public BufferedAndIterator clone() throws CloneNotSupportedException { + BufferedAndIterator answer = (BufferedAndIterator) super.clone(); + answer.buffer = this.buffer.clone(); + answer.ll = (LinkedList) this.ll.clone(); + return answer; + } + + @Override + public EWAHIterator next() { + this.buffer.clear(); + IteratorAggregation.andToContainer(this.buffer, this.buffersize * this.ll.size(), + this.ll.get(0), this.ll.get(1)); + if (this.ll.size() > 2) { + Iterator i = this.ll.iterator(); + i.next(); + i.next(); + EWAHCompressedBitmap tmpbuffer = new EWAHCompressedBitmap(); + while (i.hasNext() && this.buffer.sizeInBytes() > 0) { + IteratorAggregation.andToContainer(tmpbuffer, + this.buffer.getIteratingRLW(), i.next()); + this.buffer.swap(tmpbuffer); + tmpbuffer.clear(); + } + } + Iterator i = this.ll.iterator(); + while(i.hasNext()) { + if(i.next().size() == 0) { + this.ll.clear(); + break; + } + } + return this.buffer.getEWAHIterator(); + } + +} diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/IteratorUtil.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/IteratorUtil.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/IteratorUtil.java 1970-01-01 00:00:00.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/IteratorUtil.java 2013-11-12 14:31:20.000000000 +0000 @@ -0,0 +1,132 @@ +package com.googlecode.javaewah; + +import java.util.Iterator; + +/* + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. + */ +/** + * Convenience functions for working over iterators + * + */ +public class IteratorUtil { + + /** + * @param i iterator we wish to iterate over + * @return an iterator over the set bits corresponding to the iterator + */ + public static IntIterator toSetBitsIntIterator(final IteratingRLW i) { + return new IntIteratorOverIteratingRLW(i); + } + + /** + * @param i iterator we wish to iterate over + * @return an iterator over the set bits corresponding to the iterator + */ + public static Iterator toSetBitsIterator(final IteratingRLW i) { + return new Iterator() { + @Override + public boolean hasNext() { + return this.under.hasNext(); + } + + @Override + public Integer next() { + return new Integer(this.under.next()); + } + + @Override + public void remove() { + } + + final private IntIterator under = toSetBitsIntIterator(i); + }; + + } + + /** + * Generate a bitmap from an iterator + * + * @param i iterator we wish to materialize + * @param c where we write + */ + public static void materialize(final IteratingRLW i, final BitmapStorage c) { + while (true) { + if (i.getRunningLength() > 0) { + c.addStreamOfEmptyWords(i.getRunningBit(), i.getRunningLength()); + } + for (int k = 0; k < i.getNumberOfLiteralWords(); ++k) + c.add(i.getLiteralWordAt(k)); + if (!i.next()) + break; + } + } + + /** + * @param i iterator we wish to iterate over + * @return the cardinality (number of set bits) corresponding to the iterator + */ + public static int cardinality(final IteratingRLW i) { + int answer = 0; + while (true) { + if(i.getRunningBit()) answer += i.getRunningLength() * EWAHCompressedBitmap.wordinbits; + for (int k = 0; k < i.getNumberOfLiteralWords(); ++k) + answer += Long.bitCount(i.getLiteralWordAt(k)); + if(!i.next()) break; + } + return answer; + } + + /** + * @param x set of bitmaps + * @return an array of iterators corresponding to the array of bitmaps + */ + public static IteratingRLW[] toIterators(final EWAHCompressedBitmap... x) { + IteratingRLW[] X = new IteratingRLW[x.length]; + for (int k = 0; k < X.length; ++k) { + X[k] = new IteratingBufferedRunningLengthWord(x[k]); + } + return X; + } + /** + * Turn an iterator into a bitmap. + * + * @param i iterator we wish to materialize + * @param c where we write + * @param Max maximum number of words we wish to materialize + * @return how many words were actually materialized + */ + public static long materialize(final IteratingRLW i, final BitmapStorage c, long Max) { + final long origMax = Max; + while (true) { + if (i.getRunningLength() > 0) { + long L = i.getRunningLength(); + if(L > Max) L = Max; + c.addStreamOfEmptyWords(i.getRunningBit(), L); + Max -= L; + } + long L = i.getNumberOfLiteralWords(); + for (int k = 0; k < L; ++k) + c.add(i.getLiteralWordAt(k)); + if(Max>0) { + if (!i.next()) + break; + } + else break; + } + return origMax - Max; + } + /** + * Turn an iterator into a bitmap + * + * @param i iterator we wish to materialize + * @return materialized version of the iterator + */ + public static EWAHCompressedBitmap materialize(final IteratingRLW i) { + EWAHCompressedBitmap ewah = new EWAHCompressedBitmap(); + materialize(i, ewah); + return ewah; + } + +} diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/LogicalElement.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/LogicalElement.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/LogicalElement.java 2013-06-11 17:48:37.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/LogicalElement.java 2013-11-12 14:31:20.000000000 +0000 @@ -6,19 +6,20 @@ * be concerned by this class. * * @author Daniel Lemire + * @param the type of element (e.g., a bitmap class) * */ public interface LogicalElement { /** * Compute the bitwise logical and - * @param another element + * @param le element * @return the result of the operation */ public T and(T le); /** * Compute the bitwise logical and not - * @param another element + * @param le element * @return the result of the operation */ public T andNot(T le); @@ -27,10 +28,12 @@ * Compute the bitwise logical not (in place) */ public void not(); - @SuppressWarnings("rawtypes") + + + @SuppressWarnings({ "rawtypes", "javadoc" }) /** * Compute the bitwise logical or - * @param another element + * @param le another element * @return the result of the operation */ public LogicalElement or(T le); @@ -38,7 +41,6 @@ /** * How many logical bits does this element represent? * - * @param another element * @return the number of bits represented by this element */ public int sizeInBits(); @@ -52,7 +54,7 @@ /** * Compute the bitwise logical Xor - * @param another element + * @param le element * @return the results of the operation */ public T xor(T le); diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/NonEmptyVirtualStorage.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/NonEmptyVirtualStorage.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/NonEmptyVirtualStorage.java 2013-06-11 17:48:37.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/NonEmptyVirtualStorage.java 2013-11-12 14:31:20.000000000 +0000 @@ -1,8 +1,8 @@ package com.googlecode.javaewah; /* - * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc. and Veronika Zenz - * Licensed under APL 2.0. + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. */ /** * This is a BitmapStorage that can be used to determine quickly if the result @@ -38,42 +38,43 @@ * * @see com.googlecode.javaewah.BitmapStorage#add(long) */ - public void add(long newdata) { + @Override +public void add(long newdata) { if (newdata != 0) throw nonEmptyException; return; } /** - * throws a NonEmptyException exception when number > 0 + * throws a NonEmptyException exception when number is greater than 0 * - * @see com.googlecode.javaewah.BitmapStorage#addStreamOfLiteralWords(long[], long, long) */ - public void addStreamOfLiteralWords(long[] data, int start, int number) { + @Override +public void addStreamOfLiteralWords(long[] data, int start, int number) { if(number>0){ throw nonEmptyException; } } /** - * If the boolean value is true and number>0, then it throws a NonEmptyException exception, + * If the boolean value is true and number is greater than 0, then it throws a NonEmptyException exception, * otherwise, nothing happens. * * @see com.googlecode.javaewah.BitmapStorage#addStreamOfEmptyWords(boolean, long) */ - public void addStreamOfEmptyWords(boolean v, long number) { + @Override +public void addStreamOfEmptyWords(boolean v, long number) { if (v && (number>0)) throw nonEmptyException; return; } /** - * throws a NonEmptyException exception when number > 0 + * throws a NonEmptyException exception when number is greater than 0 * - * @see com.googlecode.javaewah.BitmapStorage#addStreamOfNegatedLiteralWords(long[], long, - * long) */ - public void addStreamOfNegatedLiteralWords(long[] data, int start, int number) { + @Override +public void addStreamOfNegatedLiteralWords(long[] data, int start, int number) { if(number>0){ throw nonEmptyException; } @@ -84,7 +85,8 @@ * * @see com.googlecode.javaewah.BitmapStorage#setSizeInBits(int) */ - public void setSizeInBits(int bits) { + @Override +public void setSizeInBits(int bits) { } } diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/RunningLengthWord.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/RunningLengthWord.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah/RunningLengthWord.java 2013-06-11 17:48:37.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah/RunningLengthWord.java 2013-11-12 14:31:20.000000000 +0000 @@ -1,139 +1,152 @@ package com.googlecode.javaewah; /* - * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc. and Veronika Zenz - * Licensed under APL 2.0. + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. */ /** * Mostly for internal use. - * + * * @since 0.1.0 * @author Daniel Lemire */ -public final class RunningLengthWord { +public final class RunningLengthWord implements Cloneable { + + /** + * Instantiates a new running length word. + * + * @param a + * an array of 64-bit words + * @param p + * position in the array where the running length word is + * located. + */ + RunningLengthWord(final EWAHCompressedBitmap a, final int p) { + this.parent = a; + this.position = p; + } + + /** + * Gets the number of literal words. + * + * @return the number of literal words + */ + public int getNumberOfLiteralWords() { + return (int) (this.parent.buffer[this.position] >>> (1 + runninglengthbits)); + } + + /** + * Gets the running bit. + * + * @return the running bit + */ + public boolean getRunningBit() { + return (this.parent.buffer[this.position] & 1) != 0; + } + + /** + * Gets the running length. + * + * @return the running length + */ + public long getRunningLength() { + return (this.parent.buffer[this.position] >>> 1) + & largestrunninglengthcount; + } + + /** + * Sets the number of literal words. + * + * @param number + * the new number of literal words + */ + public void setNumberOfLiteralWords(final long number) { + this.parent.buffer[this.position] |= notrunninglengthplusrunningbit; + this.parent.buffer[this.position] &= (number << (runninglengthbits + 1)) + | runninglengthplusrunningbit; + } + + /** + * Sets the running bit. + * + * @param b + * the new running bit + */ + public void setRunningBit(final boolean b) { + if (b) + this.parent.buffer[this.position] |= 1l; + else + this.parent.buffer[this.position] &= ~1l; + } + + /** + * Sets the running length. + * + * @param number + * the new running length + */ + public void setRunningLength(final long number) { + this.parent.buffer[this.position] |= shiftedlargestrunninglengthcount; + this.parent.buffer[this.position] &= (number << 1) + | notshiftedlargestrunninglengthcount; + } + + /** + * Return the size in uncompressed words represented by this running + * length word. + * + * @return the size + */ + public long size() { + return getRunningLength() + getNumberOfLiteralWords(); + } + + /* + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return "running bit = " + getRunningBit() + + " running length = " + getRunningLength() + + " number of lit. words " + getNumberOfLiteralWords(); + } + + @Override + public RunningLengthWord clone() throws CloneNotSupportedException { + RunningLengthWord answer; + answer = (RunningLengthWord) super.clone(); + answer.parent = this.parent; + answer.position = this.position; + return answer; + } + + /** The array of words. */ + public EWAHCompressedBitmap parent; + + /** The position in array. */ + public int position; + + /** + * number of bits dedicated to marking of the running length of clean + * words + */ + public static final int runninglengthbits = 32; + + private static final int literalbits = 64 - 1 - runninglengthbits; + + /** largest number of literal words in a run. */ + public static final int largestliteralcount = (1 << literalbits) - 1; + + /** largest number of clean words in a run */ + public static final long largestrunninglengthcount = (1l << runninglengthbits) - 1; + + private static final long runninglengthplusrunningbit = (1l << (runninglengthbits + 1)) - 1; + + private static final long shiftedlargestrunninglengthcount = largestrunninglengthcount << 1; + + private static final long notrunninglengthplusrunningbit = ~runninglengthplusrunningbit; + + private static final long notshiftedlargestrunninglengthcount = ~shiftedlargestrunninglengthcount; - /** - * Instantiates a new running length word. - * - * @param a an array of 64-bit words - * @param p position in the array where the running length word is located. - */ - RunningLengthWord(final long[] a, final int p) { - this.array = a; - this.position = p; - } - - /** - * Gets the number of literal words. - * - * @return the number of literal words - */ - public int getNumberOfLiteralWords() { - return (int) (this.array[this.position] >>> (1 + runninglengthbits)); - } - - /** - * Gets the running bit. - * - * @return the running bit - */ - public boolean getRunningBit() { - return (this.array[this.position] & 1) != 0; - } - - /** - * Gets the running length. - * - * @return the running length - */ - public long getRunningLength() { - return (this.array[this.position] >>> 1) & largestrunninglengthcount; - } - - /** - * Sets the number of literal words. - * - * @param number the new number of literal words - */ - public void setNumberOfLiteralWords(final long number) { - this.array[this.position] |= notrunninglengthplusrunningbit; - this.array[this.position] &= (number << (runninglengthbits + 1)) - | runninglengthplusrunningbit; - } - - /** - * Sets the running bit. - * - * @param b the new running bit - */ - public void setRunningBit(final boolean b) { - if (b) - this.array[this.position] |= 1l; - else - this.array[this.position] &= ~1l; - } - - /** - * Sets the running length. - * - * @param number the new running length - */ - public void setRunningLength(final long number) { - this.array[this.position] |= shiftedlargestrunninglengthcount; - this.array[this.position] &= (number << 1) - | notshiftedlargestrunninglengthcount; - } - - /** - * Return the size in uncompressed words represented by - * this running length word. - * - * @return the long - */ - public long size() { - return getRunningLength() + getNumberOfLiteralWords(); - } - - /* - * @see java.lang.Object#toString() - */ - @Override - public String toString() { - return "running bit = " + getRunningBit() + " running length = " - + getRunningLength() + " number of lit. words " - + getNumberOfLiteralWords(); - } - - - /** The array of words. */ - public long[] array; - - /** The position in array. */ - public int position; - - - - - /** number of bits dedicated to marking of the running length of clean words */ - public static final int runninglengthbits = 32; - - private static final int literalbits = 64 - 1 - runninglengthbits; - - - /** largest number of literal words in a run. */ - public static final int largestliteralcount = (1 << literalbits) - 1; - - /** largest number of clean words in a run */ - public static final long largestrunninglengthcount = (1l << runninglengthbits) - 1; - - private static final long runninglengthplusrunningbit = (1l << (runninglengthbits + 1)) - 1; - - private static final long shiftedlargestrunninglengthcount = largestrunninglengthcount << 1; - - - private static final long notrunninglengthplusrunningbit = ~runninglengthplusrunningbit; - - private static final long notshiftedlargestrunninglengthcount = ~shiftedlargestrunninglengthcount; - } \ No newline at end of file diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah32/BitCounter32.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah32/BitCounter32.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah32/BitCounter32.java 2013-06-11 17:48:37.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah32/BitCounter32.java 2013-11-12 14:31:20.000000000 +0000 @@ -4,7 +4,7 @@ /* * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc. and Veronika Zenz - * Licensed under APL 2.0. + * Licensed under the Apache License, Version 2.0. */ /** * BitCounter is a fake bitset data structure. Instead of storing the actual data, @@ -22,7 +22,8 @@ * @param newdata the word */ // @Override : causes problems with Java 1.5 - public void add(final int newdata) { + @Override +public void add(final int newdata) { this.oneBits += Integer.bitCount(newdata); } @@ -35,7 +36,8 @@ * @param number the number of literal words to add */ // @Override : causes problems with Java 1.5 - public void addStreamOfLiteralWords(int[] data, int start, int number) { + @Override +public void addStreamOfLiteralWords(int[] data, int start, int number) { for(int i=start;i iterator) { + this.masteriterator = iterator; + if(this.masteriterator.hasNext()) { + this.iterator = this.masteriterator.next(); + this.brlw = new BufferedRunningLengthWord32(this.iterator.next()); + this.literalWordStartPosition = this.iterator.literalWords() + this.brlw.literalwordoffset; + this.buffer = this.iterator.buffer(); + } + } + + + /** + * Discard first words, iterating to the next running length word if needed. + * + * @param x the number of words to be discarded + */ + @Override + public void discardFirstWords(int x) { + while (x > 0) { + if (this.brlw.RunningLength > x) { + this.brlw.RunningLength -= x; + return; + } + x -= this.brlw.RunningLength; + this.brlw.RunningLength = 0; + int toDiscard = x > this.brlw.NumberOfLiteralWords ? this.brlw.NumberOfLiteralWords : x; + + this.literalWordStartPosition += toDiscard; + this.brlw.NumberOfLiteralWords -= toDiscard; + x -= toDiscard; + if ((x > 0) || (this.brlw.size() == 0)) { + if (!this.next()) { + break; + } + } + } + } + /** + * Move to the next RunningLengthWord + * @return whether the move was possible + */ + @Override + public boolean next() { + if (!this.iterator.hasNext()) { + if(!reload()) { + this.brlw.NumberOfLiteralWords = 0; + this.brlw.RunningLength = 0; + return false; + } + } + this.brlw.reset(this.iterator.next()); + this.literalWordStartPosition = this.iterator.literalWords(); // + this.brlw.literalwordoffset ==0 + return true; + } + private boolean reload() { + if(!this.masteriterator.hasNext()) { + return false; + } + this.iterator = this.masteriterator.next(); + this.buffer = this.iterator.buffer(); + return true; + } + + + /** + * Get the nth literal word for the current running length word + * @param index zero based index + * @return the literal word + */ + @Override + public int getLiteralWordAt(int index) { + return this.buffer[this.literalWordStartPosition + index]; + } + + /** + * Gets the number of literal words for the current running length word. + * + * @return the number of literal words + */ + @Override + public int getNumberOfLiteralWords() { + return this.brlw.NumberOfLiteralWords; + } + + /** + * Gets the running bit. + * + * @return the running bit + */ + @Override + public boolean getRunningBit() { + return this.brlw.RunningBit; + } + + /** + * Gets the running length. + * + * @return the running length + */ + @Override + public int getRunningLength() { + return this.brlw.RunningLength; + } + + /** + * Size in uncompressed words of the current running length word. + * + * @return the size + */ + @Override + public int size() { + return this.brlw.size(); + } + + @Override + public BufferedIterator32 clone() throws CloneNotSupportedException { + BufferedIterator32 answer = (BufferedIterator32) super.clone(); + answer.brlw = this.brlw.clone(); + answer.buffer = this.buffer; + answer.iterator = this.iterator.clone(); + answer.literalWordStartPosition = this.literalWordStartPosition; + answer.masteriterator = this.masteriterator.clone(); + return answer; + } + + private BufferedRunningLengthWord32 brlw; + private int[] buffer; + private int literalWordStartPosition; + private EWAHIterator32 iterator; + private CloneableIterator masteriterator; + } \ No newline at end of file diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah32/BufferedRunningLengthWord32.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah32/BufferedRunningLengthWord32.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah32/BufferedRunningLengthWord32.java 2013-06-11 17:48:37.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah32/BufferedRunningLengthWord32.java 2013-11-12 14:31:20.000000000 +0000 @@ -1,8 +1,8 @@ package com.googlecode.javaewah32; /* - * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc. and Veronika Zenz - * Licensed under APL 2.0. + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. */ @@ -15,7 +15,7 @@ * @since 0.5.0 * */ -public final class BufferedRunningLengthWord32 { +public final class BufferedRunningLengthWord32 implements Cloneable { /** * Instantiates a new buffered running length word. @@ -34,13 +34,13 @@ * @param rlw the rlw */ public BufferedRunningLengthWord32(final RunningLengthWord32 rlw) { - this(rlw.array[rlw.position]); + this(rlw.parent.buffer[rlw.position]); } /** * Discard first words. * - * @param x the x + * @param x the number of words to be discarded */ public void discardFirstWords(int x) { if (this.RunningLength >= x) { @@ -99,7 +99,7 @@ * @param rlw the other running length word */ public void reset(final RunningLengthWord32 rlw) { - reset(rlw.array[rlw.position]); + reset(rlw.parent.buffer[rlw.position]); } /** @@ -147,6 +147,16 @@ + getRunningLength() + " number of lit. words " + getNumberOfLiteralWords(); } + + @Override +public BufferedRunningLengthWord32 clone() throws CloneNotSupportedException { + BufferedRunningLengthWord32 answer = (BufferedRunningLengthWord32) super.clone(); + answer.literalwordoffset = this.literalwordoffset; + answer.NumberOfLiteralWords = this.NumberOfLiteralWords; + answer.RunningBit = this.RunningBit; + answer.RunningLength = this.RunningLength; + return answer; + } /** how many literal words have we read so far? */ public int literalwordoffset = 0; diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah32/EWAHCompressedBitmap32.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah32/EWAHCompressedBitmap32.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah32/EWAHCompressedBitmap32.java 2013-06-11 17:48:37.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah32/EWAHCompressedBitmap32.java 2013-11-12 14:31:20.000000000 +0000 @@ -1,8 +1,8 @@ package com.googlecode.javaewah32; /* - * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc. and Veronika Zenz - * Licensed under APL 2.0. + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. */ import java.util.*; @@ -19,12 +19,6 @@ *

* *

- * In contrast witht the 64-bit EWAH scheme (javaewah.EWAHCompressedBitmap), you - * can expect this class to compress better, but to be slower at processing the - * data. In effect, there is a trade-off between memory usage and performances. - *

- * - *

* In contrast with the 64-bit EWAH scheme (javaewah.EWAHCompressedBitmap), you * can expect this class to compress better, but to be slower at processing the * data. In effect, there is a trade-off between memory usage and performances. @@ -44,7 +38,7 @@ * *

    *
  • Daniel Lemire, Owen Kaser, Kamel Aouiche, Sorting improves - * word-aligned bitmap indexes. Data & Knowledge Engineering 69 (1), pages + * word-aligned bitmap indexes. Data & Knowledge Engineering 69 (1), pages * 3-28, 2010. http://arxiv.org/abs/0901.3751
    • *
* @@ -58,7 +52,7 @@ */ public EWAHCompressedBitmap32() { this.buffer = new int[defaultbuffersize]; - this.rlw = new RunningLengthWord32(this.buffer, 0); + this.rlw = new RunningLengthWord32(this, 0); } /** @@ -71,19 +65,24 @@ */ public EWAHCompressedBitmap32(final int buffersize) { this.buffer = new int[buffersize]; - this.rlw = new RunningLengthWord32(this.buffer, 0); + this.rlw = new RunningLengthWord32(this, 0); } /** * Adding words directly to the bitmap (for expert use). * * This is normally how you add data to the array. So you add bits in streams - * of 8*8 bits. + * of 4*8 bits. + * + * Example: if you add 321, you are have added (in binary notation) + * 0b101000001, so you have effectively called set(0), set(6), set(8) + * in sequence. * * @param newdata * the word */ - public void add(final int newdata) { + @Override +public void add(final int newdata) { add(newdata, wordinbits); } @@ -163,13 +162,14 @@ * @param number * the number of literal words to add */ - public void addStreamOfLiteralWords(final int[] data, final int start, + @Override +public void addStreamOfLiteralWords(final int[] data, final int start, final int number) { int leftovernumber = number; while (leftovernumber > 0) { final int NumberOfLiteralWords = this.rlw.getNumberOfLiteralWords(); - final int whatwecanadd = number < RunningLengthWord32.largestliteralcount - - NumberOfLiteralWords ? number + final int whatwecanadd = leftovernumber < RunningLengthWord32.largestliteralcount + - NumberOfLiteralWords ? leftovernumber : RunningLengthWord32.largestliteralcount - NumberOfLiteralWords; this.rlw.setNumberOfLiteralWords(NumberOfLiteralWords @@ -193,7 +193,8 @@ * @param number * the number */ - public void addStreamOfEmptyWords(final boolean v, int number) { + @Override +public void addStreamOfEmptyWords(final boolean v, int number) { if (number == 0) return; this.sizeinbits += number * wordinbits; @@ -239,13 +240,14 @@ * @param number * the number of literal words to add */ - public void addStreamOfNegatedLiteralWords(final int[] data, final int start, + @Override +public void addStreamOfNegatedLiteralWords(final int[] data, final int start, final int number) { int leftovernumber = number; while (leftovernumber > 0) { final int NumberOfLiteralWords = this.rlw.getNumberOfLiteralWords(); - final int whatwecanadd = number < RunningLengthWord32.largestliteralcount - - NumberOfLiteralWords ? number + final int whatwecanadd = leftovernumber < RunningLengthWord32.largestliteralcount + - NumberOfLiteralWords ? leftovernumber : RunningLengthWord32.largestliteralcount - NumberOfLiteralWords; this.rlw.setNumberOfLiteralWords(NumberOfLiteralWords @@ -267,11 +269,15 @@ * The running time is proportional to the sum of the compressed sizes (as * reported by sizeInBytes()). * + * If you are not planning on adding to the resulting bitmap, you may call the trim() + * method to reduce memory usage. + * * @param a * the other bitmap * @return the EWAH compressed bitmap */ - public EWAHCompressedBitmap32 and(final EWAHCompressedBitmap32 a) { + @Override +public EWAHCompressedBitmap32 and(final EWAHCompressedBitmap32 a) { final EWAHCompressedBitmap32 container = new EWAHCompressedBitmap32(); container .reserve(this.actualsizeinwords > a.actualsizeinwords ? this.actualsizeinwords @@ -338,10 +344,13 @@ rlwj.discardFirstWords(nbre_literal); } } - final boolean i_remains = rlwi.size()>0; - final IteratingBufferedRunningLengthWord32 remaining = i_remains ? rlwi : rlwj; - remaining.dischargeAsEmpty(container); - container.setSizeInBits(Math.max(sizeInBits(), a.sizeInBits())); + if (adjustContainerSizeWhenAggregating) { + final boolean i_remains = rlwi.size() > 0; + final IteratingBufferedRunningLengthWord32 remaining = i_remains ? rlwi + : rlwj; + remaining.dischargeAsEmpty(container); + container.setSizeInBits(Math.max(sizeInBits(), a.sizeInBits())); + } } @@ -367,11 +376,15 @@ * The running time is proportional to the sum of the compressed sizes (as * reported by sizeInBytes()). * + * If you are not planning on adding to the resulting bitmap, you may call the trim() + * method to reduce memory usage. + * * @param a * the other bitmap * @return the EWAH compressed bitmap */ - public EWAHCompressedBitmap32 andNot(final EWAHCompressedBitmap32 a) { + @Override +public EWAHCompressedBitmap32 andNot(final EWAHCompressedBitmap32 a) { final EWAHCompressedBitmap32 container = new EWAHCompressedBitmap32(); container .reserve(this.actualsizeinwords > a.actualsizeinwords ? this.actualsizeinwords @@ -387,9 +400,8 @@ * The running time is proportional to the sum of the compressed sizes (as * reported by sizeInBytes()). * - * @param a - * the other bitmap - * @return the EWAH compressed bitmap + * @param a the other bitmap + * @param container where we store the result */ public void andNotToContainer(final EWAHCompressedBitmap32 a, final BitmapStorage32 container) { @@ -434,9 +446,10 @@ final IteratingBufferedRunningLengthWord32 remaining = i_remains ? rlwi : rlwj; if(i_remains) remaining.discharge(container); - else + else if (adjustContainerSizeWhenAggregating) remaining.dischargeAsEmpty(container); - container.setSizeInBits(Math.max(sizeInBits(), a.sizeInBits())); + if (adjustContainerSizeWhenAggregating) + container.setSizeInBits(Math.max(sizeInBits(), a.sizeInBits())); } @@ -463,7 +476,7 @@ */ public int cardinality() { int counter = 0; - final EWAHIterator32 i = new EWAHIterator32(this.buffer, + final EWAHIterator32 i = new EWAHIterator32(this, this.actualsizeinwords); while (i.hasNext()) { RunningLengthWord32 localrlw = i.next(); @@ -493,7 +506,7 @@ * @see java.lang.Object#clone() */ @Override - public Object clone() throws java.lang.CloneNotSupportedException { + public EWAHCompressedBitmap32 clone() throws java.lang.CloneNotSupportedException { final EWAHCompressedBitmap32 clone = (EWAHCompressedBitmap32) super.clone(); clone.buffer = this.buffer.clone(); clone.actualsizeinwords = this.actualsizeinwords; @@ -517,13 +530,12 @@ } for (int k = 0; k < this.actualsizeinwords; ++k) this.buffer[k] = in.readInt(); - this.rlw = new RunningLengthWord32(this.buffer, in.readInt()); + this.rlw = new RunningLengthWord32(this, in.readInt()); } /** * Check to see whether the two compressed bitmaps contain the same set bits. * - * @author Colby Ranger * @see java.lang.Object#equals(java.lang.Object) */ @Override @@ -589,8 +601,15 @@ * @return the EWAHIterator */ public EWAHIterator32 getEWAHIterator() { - return new EWAHIterator32(this.buffer, this.actualsizeinwords); + return new EWAHIterator32(this, this.actualsizeinwords); } + + /** + * @return the IteratingRLW iterator corresponding to this bitmap + */ + public IteratingRLW32 getIteratingRLW() { + return new IteratingBufferedRunningLengthWord32(this); + } /** * get the locations of the true values as one vector. (may use more memory @@ -600,7 +619,7 @@ */ public List getPositions() { final ArrayList v = new ArrayList(); - final EWAHIterator32 i = new EWAHIterator32(this.buffer, + final EWAHIterator32 i = new EWAHIterator32(this, this.actualsizeinwords); int pos = 0; while (i.hasNext()) { @@ -638,7 +657,7 @@ public int hashCode() { int karprabin = 0; final int B = 31; - final EWAHIterator32 i = new EWAHIterator32(this.buffer, + final EWAHIterator32 i = new EWAHIterator32(this, this.actualsizeinwords); while( i.hasNext() ) { i.next(); @@ -681,7 +700,7 @@ */ public IntIterator intIterator() { return new IntIteratorImpl32( - new EWAHIterator32(this.buffer, this.actualsizeinwords)); + new EWAHIterator32(this, this.actualsizeinwords)); } /** @@ -690,17 +709,21 @@ * * @return the iterator */ - public Iterator iterator() { + @Override +public Iterator iterator() { return new Iterator() { - public boolean hasNext() { + @Override + public boolean hasNext() { return this.under.hasNext(); } - public Integer next() { + @Override + public Integer next() { return new Integer(this.under.next()); } - public void remove() { + @Override + public void remove() { throw new UnsupportedOperationException("bitsets do not support remove"); } @@ -722,9 +745,14 @@ final int number) { while (this.actualsizeinwords + number >= this.buffer.length) { final int oldbuffer[] = this.buffer; - this.buffer = new int[oldbuffer.length * 2]; + if(this.actualsizeinwords + number < 32768) + this.buffer = new int[(this.actualsizeinwords + number) * 2]; + else if ((this.actualsizeinwords + number) * 3 / 2 < this.actualsizeinwords + number) + this.buffer = new int[Integer.MAX_VALUE]; + else + this.buffer = new int[(this.actualsizeinwords + number) * 3 / 2]; System.arraycopy(oldbuffer, 0, this.buffer, 0, oldbuffer.length); - this.rlw.array = this.buffer; + this.rlw.parent.buffer = this.buffer; } for (int k = 0; k < number; ++k) this.buffer[this.actualsizeinwords + k] = ~data[start + k]; @@ -739,8 +767,9 @@ * sizeInBytes()). * */ - public void not() { - final EWAHIterator32 i = new EWAHIterator32(this.buffer, + @Override +public void not() { + final EWAHIterator32 i = new EWAHIterator32(this, this.actualsizeinwords); if (!i.hasNext()) return; @@ -769,51 +798,23 @@ } } - public int oldaddStreamOfEmptyWords(final boolean v, final int number) { - if (number == 0) - return 0; - final boolean noliteralword = (this.rlw.getNumberOfLiteralWords() == 0); - final int runlen = this.rlw.getRunningLength(); - if ((noliteralword) && (runlen == 0)) { - this.rlw.setRunningBit(v); - } - int wordsadded = 0; - if ((noliteralword) && (this.rlw.getRunningBit() == v) - && (runlen < RunningLengthWord32.largestrunninglengthcount)) { - int whatwecanadd = number < RunningLengthWord32.largestrunninglengthcount - - runlen ? number : RunningLengthWord32.largestrunninglengthcount - - runlen; - this.rlw.setRunningLength(runlen + whatwecanadd); - this.sizeinbits += whatwecanadd * wordinbits; - if (number - whatwecanadd > 0) - addStreamOfEmptyWords(v, number - whatwecanadd); - } else { - push_back(0); - ++wordsadded; - this.rlw.position = this.actualsizeinwords - 1; - final int whatwecanadd = number < RunningLengthWord32.largestrunninglengthcount ? number - : RunningLengthWord32.largestrunninglengthcount; - this.rlw.setRunningBit(v); - this.rlw.setRunningLength(whatwecanadd); - this.sizeinbits += whatwecanadd * wordinbits; - if (number - whatwecanadd > 0) - addStreamOfEmptyWords(v, number - whatwecanadd); - } - return wordsadded; - } - + /** * Returns a new compressed bitmap containing the bitwise OR values of the * current bitmap with some other bitmap. * * The running time is proportional to the sum of the compressed sizes (as * reported by sizeInBytes()). - * + * + * If you are not planning on adding to the resulting bitmap, you may call the trim() + * method to reduce memory usage. + * * @param a * the other bitmap * @return the EWAH compressed bitmap */ - public EWAHCompressedBitmap32 or(final EWAHCompressedBitmap32 a) { + @Override +public EWAHCompressedBitmap32 or(final EWAHCompressedBitmap32 a) { final EWAHCompressedBitmap32 container = new EWAHCompressedBitmap32(); container.reserve(this.actualsizeinwords + a.actualsizeinwords); orToContainer(a, container); @@ -892,9 +893,14 @@ private void push_back(final int data) { if (this.actualsizeinwords == this.buffer.length) { final int oldbuffer[] = this.buffer; - this.buffer = new int[oldbuffer.length * 2]; + if(oldbuffer.length < 32768) + this.buffer = new int[oldbuffer.length * 2]; + else if (oldbuffer.length * 3 / 2 < oldbuffer.length) + this.buffer = new int[Integer.MAX_VALUE]; + else + this.buffer = new int[oldbuffer.length * 3 / 2]; System.arraycopy(oldbuffer, 0, this.buffer, 0, oldbuffer.length); - this.rlw.array = this.buffer; + this.rlw.parent.buffer = this.buffer; } this.buffer[this.actualsizeinwords++] = data; } @@ -910,11 +916,16 @@ * the number of words to add */ private void push_back(final int[] data, final int start, final int number) { - while (this.actualsizeinwords + number >= this.buffer.length) { + if (this.actualsizeinwords + number >= this.buffer.length) { final int oldbuffer[] = this.buffer; - this.buffer = new int[oldbuffer.length * 2]; + if(this.actualsizeinwords + number < 32768) + this.buffer = new int[(this.actualsizeinwords + number) * 2]; + else if((this.actualsizeinwords + number) * 3 / 2 < this.actualsizeinwords + number) //overflow + this.buffer = new int[Integer.MAX_VALUE]; + else + this.buffer = new int[(this.actualsizeinwords + number) * 3 / 2]; System.arraycopy(oldbuffer, 0, this.buffer, 0, oldbuffer.length); - this.rlw.array = this.buffer; + this.rlw.parent.buffer = this.buffer; } System.arraycopy(data, start, this.buffer, this.actualsizeinwords, number); this.actualsizeinwords += number; @@ -923,7 +934,8 @@ /* * @see java.io.Externalizable#readExternal(java.io.ObjectInput) */ - public void readExternal(ObjectInput in) throws IOException { + @Override +public void readExternal(ObjectInput in) throws IOException { deserialize(in); } @@ -939,7 +951,7 @@ final int oldbuffer[] = this.buffer; this.buffer = new int[size]; System.arraycopy(oldbuffer, 0, this.buffer, 0, oldbuffer.length); - this.rlw.array = this.buffer; + this.rlw.parent.buffer = this.buffer; return true; } return false; @@ -969,15 +981,48 @@ public int serializedSizeInBytes() { return this.sizeInBytes() + 3 * 4; } + + /** + * Query the value of a single bit. Relying on this method when speed is + * needed is discouraged. The complexity is linear with the size of the + * bitmap. + * + * (This implementation is based on zhenjl's Go version of JavaEWAH.) + * + * @param i + * the bit we are interested in + * @return whether the bit is set to true + */ + public boolean get(final int i) { + if ((i < 0) || (i >= this.sizeinbits)) + return false; + int WordChecked = 0; + final IteratingRLW32 j = getIteratingRLW(); + final int wordi = i / wordinbits; + while (WordChecked <= wordi) { + WordChecked += j.getRunningLength(); + if (wordi < WordChecked) { + return j.getRunningBit(); + } + if (wordi < WordChecked + j.getNumberOfLiteralWords()) { + final int w = j.getLiteralWordAt(wordi + - WordChecked); + return (w & (1 << i)) != 0; + } + WordChecked += j.getNumberOfLiteralWords(); + j.next(); + } + return false; + } /** - * set the bit at position i to true, the bits must be set in increasing + * Set the bit at position i to true, the bits must be set in (strictly) increasing * order. For example, set(15) and then set(7) will fail. You must do set(7) * and then set(15). * * @param i * the index - * @return true if the value was set (always true when i>= sizeInBits()). + * @return true if the value was set (always true when i is greater or equal to sizeInBits()). * @throws IndexOutOfBoundsException * if i is negative or greater than Integer.MAX_VALUE - 32 */ @@ -1015,16 +1060,20 @@ } /** - * set the size in bits + * Set the size in bits. This does not change the compressed bitmap. * */ - public void setSizeInBits(final int size) { - this.sizeinbits = size; + @Override +public void setSizeInBits(final int size) { + if((size+EWAHCompressedBitmap32.wordinbits-1)/EWAHCompressedBitmap32.wordinbits!= (this.sizeinbits+EWAHCompressedBitmap32.wordinbits-1)/EWAHCompressedBitmap32.wordinbits) + throw new RuntimeException("You can only reduce the size of the bitmap within the scope of the last word. To extend the bitmap, please call setSizeInbits(int,boolean): "+size+" "+this.sizeinbits); + this.sizeinbits = size; } /** * Change the reported size in bits of the *uncompressed* bitmap represented - * by this compressed bitmap. It is not possible to reduce the sizeInBits, but + * by this compressed bitmap. It may change the underlying compressed bitmap. + * It is not possible to reduce the sizeInBits, but * it can be extended. The new bits are set to false or true depending on the * value of defaultvalue. * @@ -1062,7 +1111,8 @@ * * @return the size in bits */ - public int sizeInBits() { + @Override +public int sizeInBits() { return this.sizeinbits; } @@ -1072,7 +1122,8 @@ * * @return the size in bytes */ - public int sizeInBytes() { + @Override +public int sizeInBytes() { return this.actualsizeinwords * (wordinbits / 8); } @@ -1086,7 +1137,7 @@ int[] ans = new int[this.cardinality()]; int inanspos = 0; int pos = 0; - final EWAHIterator32 i = new EWAHIterator32(this.buffer, + final EWAHIterator32 i = new EWAHIterator32(this, this.actualsizeinwords); while (i.hasNext()) { RunningLengthWord32 localrlw = i.next(); @@ -1129,7 +1180,7 @@ public String toDebugString() { String ans = " EWAHCompressedBitmap, size in bits = " + this.sizeinbits + " size in words = " + this.actualsizeinwords + "\n"; - final EWAHIterator32 i = new EWAHIterator32(this.buffer, + final EWAHIterator32 i = new EWAHIterator32(this, this.actualsizeinwords); while (i.hasNext()) { RunningLengthWord32 localrlw = i.next(); @@ -1166,11 +1217,42 @@ answer.append("}"); return answer.toString(); } + /** + * swap the content of the bitmap with another. + * + * @param other + * bitmap to swap with + */ + public void swap(final EWAHCompressedBitmap32 other) { + int[] tmp = this.buffer; + this.buffer = other.buffer; + other.buffer = tmp; + + int tmp2 = this.rlw.position; + this.rlw.position = other.rlw.position; + other.rlw.position = tmp2; + + int tmp3 = this.actualsizeinwords; + this.actualsizeinwords = other.actualsizeinwords; + other.actualsizeinwords = tmp3; + + int tmp4 = this.sizeinbits; + this.sizeinbits = other.sizeinbits; + other.sizeinbits = tmp4; + } + /** + * Reduce the internal buffer to its minimal allowable size (given + * by this.actualsizeinwords). This can free memory. + */ + public void trim() { + this.buffer = Arrays.copyOf(this.buffer, this.actualsizeinwords); + } /* * @see java.io.Externalizable#writeExternal(java.io.ObjectOutput) */ - public void writeExternal(ObjectOutput out) throws IOException { + @Override +public void writeExternal(ObjectOutput out) throws IOException { serialize(out); } @@ -1181,11 +1263,15 @@ * The running time is proportional to the sum of the compressed sizes (as * reported by sizeInBytes()). * + * If you are not planning on adding to the resulting bitmap, you may call the trim() + * method to reduce memory usage. + * * @param a * the other bitmap * @return the EWAH compressed bitmap */ - public EWAHCompressedBitmap32 xor(final EWAHCompressedBitmap32 a) { + @Override +public EWAHCompressedBitmap32 xor(final EWAHCompressedBitmap32 a) { final EWAHCompressedBitmap32 container = new EWAHCompressedBitmap32(); container.reserve(this.actualsizeinwords + a.actualsizeinwords); xorToContainer(a, container); @@ -1270,118 +1356,20 @@ */ public static void andWithContainer(final BitmapStorage32 container, final EWAHCompressedBitmap32... bitmaps) { - if (bitmaps.length == 2) { - // should be more efficient - bitmaps[0].andToContainer(bitmaps[1], container); - return; - } - - // Sort the bitmaps in ascending order by sizeinbits. When we exhaust the - // first bitmap the rest - // of the result is zeros. - final EWAHCompressedBitmap32[] sortedBitmaps = bitmaps.clone(); - Arrays.sort(sortedBitmaps, new Comparator() { - public int compare(EWAHCompressedBitmap32 a, EWAHCompressedBitmap32 b) { - return a.sizeinbits < b.sizeinbits ? -1 - : a.sizeinbits == b.sizeinbits ? 0 : 1; - } - }); - - int maxSize = sortedBitmaps[sortedBitmaps.length - 1].sizeinbits; - - final IteratingBufferedRunningLengthWord32[] rlws = new IteratingBufferedRunningLengthWord32[bitmaps.length]; - for (int i = 0; i < sortedBitmaps.length; i++) { - EWAHIterator32 iterator = sortedBitmaps[i].getEWAHIterator(); - if (iterator.hasNext()) { - rlws[i] = new IteratingBufferedRunningLengthWord32(iterator); - } else { - // this never happens... - if (maxSize > 0) { - extendEmptyBits(container, 0, maxSize); - } - container.setSizeInBits(maxSize); - return; - } - } - - while (true) { - int maxZeroRl = 0; - int minOneRl = Integer.MAX_VALUE; - int minSize = Integer.MAX_VALUE; - int numEmptyRl = 0; - - for (IteratingBufferedRunningLengthWord32 rlw : rlws) { - int size = rlw.size(); - minSize = Math.min(minSize, size); - - if (!rlw.getRunningBit()) { - int rl = rlw.getRunningLength(); - maxZeroRl = Math.max(maxZeroRl, rl); - minOneRl = 0; - if (rl == 0 && size > 0) { - numEmptyRl++; - } - } else { - int rl = rlw.getRunningLength(); - minOneRl = Math.min(minOneRl, rl); - if (rl == 0 && size > 0) { - numEmptyRl++; - } - } - } - if (minSize == 0) { - extendEmptyBits(container, sortedBitmaps[0].sizeinbits, maxSize); - break; - } - if (maxZeroRl > 0) { - container.addStreamOfEmptyWords(false, maxZeroRl); - for (IteratingBufferedRunningLengthWord32 rlw : rlws) { - rlw.discardFirstWords(maxZeroRl); - } - } else if (minOneRl > 0) { - container.addStreamOfEmptyWords(true, minOneRl); - for (IteratingBufferedRunningLengthWord32 rlw : rlws) { - rlw.discardFirstWords(minOneRl); - } - } else { - int index = 0; - - if (numEmptyRl == 1) { - // if one rlw has literal words to process and the rest have a run of - // 1's we can write them out here - IteratingBufferedRunningLengthWord32 emptyRl = null; - int minNonEmptyRl = Integer.MAX_VALUE; - for (IteratingBufferedRunningLengthWord32 rlw : rlws) { - int rl = rlw.getRunningLength(); - if (rl == 0) { - assert emptyRl == null; - emptyRl = rlw; - } else { - minNonEmptyRl = Math.min(minNonEmptyRl, rl); - } - } - int wordsToWrite = minNonEmptyRl > minSize ? minSize : minNonEmptyRl; - if (emptyRl != null) - emptyRl.writeLiteralWords(wordsToWrite, container); - index += wordsToWrite; - } - - while (index < minSize) { - int word = ~0; - for (IteratingBufferedRunningLengthWord32 rlw : rlws) { - if (rlw.getRunningLength() <= index) { - word &= rlw.getLiteralWordAt(index - rlw.getRunningLength()); - } - } - container.add(word); - index++; - } - for (IteratingBufferedRunningLengthWord32 rlw : rlws) { - rlw.discardFirstWords(minSize); - } - } - } - container.setSizeInBits(maxSize); + if(bitmaps.length == 1) throw new IllegalArgumentException("Need at least one bitmap"); + if(bitmaps.length == 2) { + bitmaps[0].andToContainer(bitmaps[1],container); + return; + } + EWAHCompressedBitmap32 answer = new EWAHCompressedBitmap32(); + EWAHCompressedBitmap32 tmp = new EWAHCompressedBitmap32(); + bitmaps[0].andToContainer(bitmaps[1], answer); + for(int k = 2; k < bitmaps.length - 1; ++k) { + answer.andToContainer(bitmaps[k], tmp); + tmp.swap(answer); + tmp.clear(); + } + answer.andToContainer(bitmaps[bitmaps.length - 1], container); } /** @@ -1390,20 +1378,28 @@ * * It may or may not be faster than doing the aggregation two-by-two (A.and(B).and(C)). * + * If only one bitmap is provided, it is returned as is. + * + * If you are not planning on adding to the resulting bitmap, you may call the trim() + * method to reduce memory usage. + * * @param bitmaps * bitmaps to AND together * @return result of the AND */ public static EWAHCompressedBitmap32 and( final EWAHCompressedBitmap32... bitmaps) { - final EWAHCompressedBitmap32 container = new EWAHCompressedBitmap32(); - int largestSize = 0; - for (EWAHCompressedBitmap32 bitmap : bitmaps) { - largestSize = Math.max(bitmap.actualsizeinwords, largestSize); - } - container.reserve((int) (largestSize * 1.5)); - andWithContainer(container, bitmaps); - return container; + if(bitmaps.length == 1) return bitmaps[0]; + if(bitmaps.length == 2) return bitmaps[0].and(bitmaps[1]); + EWAHCompressedBitmap32 answer = new EWAHCompressedBitmap32(); + EWAHCompressedBitmap32 tmp = new EWAHCompressedBitmap32(); + bitmaps[0].andToContainer(bitmaps[1], answer); + for(int k = 2; k < bitmaps.length; ++k) { + answer.andToContainer(bitmaps[k], tmp); + tmp.swap(answer); + tmp.clear(); + } + return answer; } /** @@ -1416,6 +1412,7 @@ * @return the cardinality */ public static int andCardinality(final EWAHCompressedBitmap32... bitmaps) { + if(bitmaps.length == 1) return bitmaps[0].cardinality(); final BitCounter32 counter = new BitCounter32(); andWithContainer(counter, bitmaps); return counter.getCount(); @@ -1465,147 +1462,68 @@ /** * For internal use. Computes the bitwise or of the provided bitmaps and * stores the result in the container. + * @param container where store the result + * @param bitmaps to be aggregated */ - public static void orWithContainer(final BitmapStorage32 container, - final EWAHCompressedBitmap32... bitmaps) { - if (bitmaps.length == 2) { - // should be more efficient - bitmaps[0].orToContainer(bitmaps[1], container); - return; - } - - // Sort the bitmaps in descending order by sizeinbits. We will exhaust the - // sorted bitmaps from right to left. - final EWAHCompressedBitmap32[] sortedBitmaps = bitmaps.clone(); - Arrays.sort(sortedBitmaps, new Comparator() { - public int compare(EWAHCompressedBitmap32 a, EWAHCompressedBitmap32 b) { - return a.sizeinbits < b.sizeinbits ? 1 - : a.sizeinbits == b.sizeinbits ? 0 : -1; - } - }); - - final IteratingBufferedRunningLengthWord32[] rlws = new IteratingBufferedRunningLengthWord32[bitmaps.length]; - int maxAvailablePos = 0; - for (EWAHCompressedBitmap32 bitmap : sortedBitmaps) { - EWAHIterator32 iterator = bitmap.getEWAHIterator(); - if (iterator.hasNext()) { - rlws[maxAvailablePos++] = new IteratingBufferedRunningLengthWord32( - iterator); - } - } - - if (maxAvailablePos == 0) { // this never happens... - container.setSizeInBits(0); - return; - } - - int maxSize = sortedBitmaps[0].sizeinbits; - - while (true) { - int maxOneRl = 0; - int minZeroRl = Integer.MAX_VALUE; - int minSize = Integer.MAX_VALUE; - int numEmptyRl = 0; - for (int i = 0; i < maxAvailablePos; i++) { - IteratingBufferedRunningLengthWord32 rlw = rlws[i]; - int size = rlw.size(); - if (size == 0) { - maxAvailablePos = i; - break; - } - minSize = Math.min(minSize, size); - - if (rlw.getRunningBit()) { - int rl = rlw.getRunningLength(); - maxOneRl = Math.max(maxOneRl, rl); - minZeroRl = 0; - if (rl == 0 && size > 0) { - numEmptyRl++; - } - } else { - int rl = rlw.getRunningLength(); - minZeroRl = Math.min(minZeroRl, rl); - if (rl == 0 && size > 0) { - numEmptyRl++; - } - } - } - - if (maxAvailablePos == 0) { - break; - } else if (maxAvailablePos == 1) { - // only one bitmap is left so just write the rest of it out - rlws[0].discharge(container); - break; - } - - if (maxOneRl > 0) { - container.addStreamOfEmptyWords(true, maxOneRl); - for (int i = 0; i < maxAvailablePos; i++) { - IteratingBufferedRunningLengthWord32 rlw = rlws[i]; - rlw.discardFirstWords(maxOneRl); - } - } else if (minZeroRl > 0) { - container.addStreamOfEmptyWords(false, minZeroRl); - for (int i = 0; i < maxAvailablePos; i++) { - IteratingBufferedRunningLengthWord32 rlw = rlws[i]; - rlw.discardFirstWords(minZeroRl); - } - } else { - int index = 0; - - if (numEmptyRl == 1) { - // if one rlw has literal words to process and the rest have a run of - // 0's we can write them out here - IteratingBufferedRunningLengthWord32 emptyRl = null; - int minNonEmptyRl = Integer.MAX_VALUE; - for (int i = 0; i < maxAvailablePos; i++) { - IteratingBufferedRunningLengthWord32 rlw = rlws[i]; - int rl = rlw.getRunningLength(); - if (rl == 0) { - assert emptyRl == null; - emptyRl = rlw; - } else { - minNonEmptyRl = Math.min(minNonEmptyRl, rl); - } - } - int wordsToWrite = minNonEmptyRl > minSize ? minSize : minNonEmptyRl; - if (emptyRl != null) - emptyRl.writeLiteralWords(wordsToWrite, container); - index += wordsToWrite; - } + public static void orWithContainer(final BitmapStorage32 container, + final EWAHCompressedBitmap32... bitmaps) { + if (bitmaps.length < 2) + throw new IllegalArgumentException("You should provide at least two bitmaps, provided "+bitmaps.length); + int size = 0; + int sinbits = 0; + for (EWAHCompressedBitmap32 b : bitmaps) { + size += b.sizeInBytes(); + if (sinbits < b.sizeInBits()) + sinbits = b.sizeInBits(); + } + if (size * 8 > sinbits) { + FastAggregation32.bufferedorWithContainer(container, 65536, bitmaps); + } else { + FastAggregation32.orToContainer(container, bitmaps); + } + } - while (index < minSize) { - int word = 0; - for (int i = 0; i < maxAvailablePos; i++) { - IteratingBufferedRunningLengthWord32 rlw = rlws[i]; - if (rlw.getRunningLength() <= index) { - word |= rlw.getLiteralWordAt(index - rlw.getRunningLength()); - } - } - container.add(word); - index++; - } - for (int i = 0; i < maxAvailablePos; i++) { - IteratingBufferedRunningLengthWord32 rlw = rlws[i]; - rlw.discardFirstWords(minSize); - } - } - } - container.setSizeInBits(maxSize); - } + /** + * For internal use. Computes the bitwise xor of the provided bitmaps and + * stores the result in the container. + * @param container where store the result + * @param bitmaps to be aggregated + */ + public static void xorWithContainer(final BitmapStorage32 container, + final EWAHCompressedBitmap32... bitmaps) { + if (bitmaps.length < 2) + throw new IllegalArgumentException("You should provide at least two bitmaps, provided "+bitmaps.length); + int size = 0; + int sinbits = 0; + for (EWAHCompressedBitmap32 b : bitmaps) { + size += b.sizeInBytes(); + if (sinbits < b.sizeInBits()) + sinbits = b.sizeInBits(); + } + if (size * 8 > sinbits) { + FastAggregation32.bufferedxorWithContainer(container, 65536, bitmaps); + } else { + FastAggregation32.xorToContainer(container, bitmaps); + } + } /** * Returns a new compressed bitmap containing the bitwise OR values of the * provided bitmaps. This is typically faster than doing the aggregation * two-by-two (A.or(B).or(C).or(D)). - * + * + * If only one bitmap is provided, it is returned as is. + * + * If you are not planning on adding to the resulting bitmap, you may call the trim() + * method to reduce memory usage. + * * @param bitmaps * bitmaps to OR together * @return result of the OR */ public static EWAHCompressedBitmap32 or( final EWAHCompressedBitmap32... bitmaps) { + if(bitmaps.length == 1) return bitmaps[0]; final EWAHCompressedBitmap32 container = new EWAHCompressedBitmap32(); int largestSize = 0; for (EWAHCompressedBitmap32 bitmap : bitmaps) { @@ -1616,6 +1534,34 @@ return container; } + + /** + * Returns a new compressed bitmap containing the bitwise XOR values of the + * provided bitmaps. This is typically faster than doing the aggregation + * two-by-two (A.xor(B).xor(C).xor(D)). + * + * If only one bitmap is provided, it is returned as is. + * + * If you are not planning on adding to the resulting bitmap, you may call the trim() + * method to reduce memory usage. + * + * @param bitmaps + * bitmaps to XOR together + * @return result of the XOR + */ + public static EWAHCompressedBitmap32 xor( + final EWAHCompressedBitmap32... bitmaps) { + if(bitmaps.length == 1) return bitmaps[0]; + final EWAHCompressedBitmap32 container = new EWAHCompressedBitmap32(); + int largestSize = 0; + for (EWAHCompressedBitmap32 bitmap : bitmaps) { + largestSize = Math.max(bitmap.actualsizeinwords, largestSize); + } + container.reserve((int) (largestSize * 1.5)); + xorWithContainer(container, bitmaps); + return container; + } + /** * Returns the cardinality of the result of a bitwise OR of the values of the * provided bitmaps. Avoids needing to allocate an intermediate bitmap to hold @@ -1626,6 +1572,7 @@ * @return the cardinality */ public static int orCardinality(final EWAHCompressedBitmap32... bitmaps) { + if(bitmaps.length == 1) return bitmaps[0].cardinality(); final BitCounter32 counter = new BitCounter32(); orWithContainer(counter, bitmaps); return counter.getCount(); @@ -1652,6 +1599,9 @@ /** optimization option **/ public static final boolean usetrailingzeros = true; + /** whether we adjust after some aggregation by adding in zeroes **/ + public static final boolean adjustContainerSizeWhenAggregating = true; + /** The Constant wordinbits represents the number of bits in a int. */ public static final int wordinbits = 32; diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah32/EWAHIterator32.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah32/EWAHIterator32.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah32/EWAHIterator32.java 2013-06-11 17:48:37.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah32/EWAHIterator32.java 2013-11-12 14:31:20.000000000 +0000 @@ -2,8 +2,8 @@ /* - * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc. and Veronika Zenz - * Licensed under APL 2.0. + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. */ /** @@ -14,7 +14,7 @@ * @since 0.5.0 * */ -public final class EWAHIterator32 { +public final class EWAHIterator32 implements Cloneable { /** * Instantiates a new eWAH iterator. @@ -22,7 +22,7 @@ * @param a the array of words * @param sizeinwords the number of words that are significant in the array of words */ - public EWAHIterator32(final int[] a, final int sizeinwords) { + public EWAHIterator32(final EWAHCompressedBitmap32 a, final int sizeinwords) { this.rlw = new RunningLengthWord32(a, 0); this.size = sizeinwords; this.pointer = 0; @@ -31,7 +31,7 @@ /** * Allow expert developers to instantiate an EWAHIterator. * - * @param bitmap + * @param bitmap we want to iterate over * @return an iterator */ public static EWAHIterator32 getEWAHIterator(EWAHCompressedBitmap32 bitmap) { @@ -44,7 +44,7 @@ * @return the int[] */ public int[] buffer() { - return this.rlw.array; + return this.rlw.parent.buffer; } /** @@ -76,6 +76,15 @@ return this.rlw; } + @Override + public EWAHIterator32 clone() throws CloneNotSupportedException { + EWAHIterator32 ans = (EWAHIterator32) super.clone(); + ans.rlw = this.rlw.clone(); + ans.size = this.size; + ans.pointer = this.pointer; + return ans; + } + /** The pointer represent the location of the current running length * word in the array of words (embedded in the rlw attribute). */ int pointer; diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah32/FastAggregation32.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah32/FastAggregation32.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah32/FastAggregation32.java 1970-01-01 00:00:00.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah32/FastAggregation32.java 2013-11-12 14:31:20.000000000 +0000 @@ -0,0 +1,377 @@ +package com.googlecode.javaewah32; + +import java.util.Arrays; +import java.util.Comparator; +import java.util.PriorityQueue; + + +/** + * Fast algorithms to aggregate many bitmaps. These algorithms are just given as + * reference. They may not be faster than the corresponding methods in the + * EWAHCompressedBitmap class. + * + * @author Daniel Lemire + * + */ +public class FastAggregation32 { + + /** + * Compute the and aggregate using a temporary uncompressed bitmap. + * @param bitmaps the source bitmaps + * @param bufsize buffer size used during the computation in 64-bit words (per input bitmap) + * @return the or aggregate. + */ + public static EWAHCompressedBitmap32 bufferedand(final int bufsize, + final EWAHCompressedBitmap32... bitmaps) { + EWAHCompressedBitmap32 answer = new EWAHCompressedBitmap32(); + bufferedandWithContainer(answer,bufsize, bitmaps); + return answer; + } + /** + * Compute the and aggregate using a temporary uncompressed bitmap. + * + * @param container where the aggregate is written + * @param bufsize buffer size used during the computation in 64-bit words (per input bitmap) + * @param bitmaps the source bitmaps + */ + public static void bufferedandWithContainer(final BitmapStorage32 container,final int bufsize, + final EWAHCompressedBitmap32... bitmaps) { + + java.util.LinkedList al = new java.util.LinkedList(); + for (EWAHCompressedBitmap32 bitmap : bitmaps) { + al.add(new IteratingBufferedRunningLengthWord32(bitmap)); + } + int[] hardbitmap = new int[bufsize*bitmaps.length]; + + for(IteratingRLW32 i : al) + if (i.size() == 0) { + al.clear(); + break; + } + + while (!al.isEmpty()) { + Arrays.fill(hardbitmap, ~0); + int effective = Integer.MAX_VALUE; + for(IteratingRLW32 i : al) { + int eff = IteratorAggregation32.inplaceand(hardbitmap, i); + if (eff < effective) + effective = eff; + } + for (int k = 0; k < effective; ++k) + container.add(hardbitmap[k]); + for(IteratingRLW32 i : al) + if (i.size() == 0) { + al.clear(); + break; + } + } + } + + /** + * Compute the or aggregate using a temporary uncompressed bitmap. + * @param bitmaps the source bitmaps + * @param bufsize buffer size used during the computation in 64-bit words + * @return the or aggregate. + */ + public static EWAHCompressedBitmap32 bufferedor(final int bufsize, + final EWAHCompressedBitmap32... bitmaps) { + EWAHCompressedBitmap32 answer = new EWAHCompressedBitmap32(); + bufferedorWithContainer(answer, bufsize, bitmaps); + return answer; + } + + /** + * Compute the or aggregate using a temporary uncompressed bitmap. + * + * @param container where the aggregate is written + * @param bufsize buffer size used during the computation in 64-bit words + * @param bitmaps the source bitmaps + */ + public static void bufferedorWithContainer(final BitmapStorage32 container,final int bufsize, + final EWAHCompressedBitmap32... bitmaps) { + int range = 0; + EWAHCompressedBitmap32[] sbitmaps = bitmaps.clone(); + Arrays.sort(sbitmaps, new Comparator() { + @Override + public int compare(EWAHCompressedBitmap32 a, EWAHCompressedBitmap32 b) { + return b.sizeinbits - a.sizeinbits; + } + }); + + java.util.ArrayList al = new java.util.ArrayList(); + for (EWAHCompressedBitmap32 bitmap : sbitmaps) { + if (bitmap.sizeinbits > range) + range = bitmap.sizeinbits; + al.add(new IteratingBufferedRunningLengthWord32(bitmap)); + } + int[] hardbitmap = new int[bufsize]; + int maxr = al.size(); + while (maxr > 0) { + int effective = 0; + for (int k = 0; k < maxr; ++k) { + if (al.get(k).size() > 0) { + int eff = IteratorAggregation32.inplaceor(hardbitmap, al.get(k)); + if (eff > effective) + effective = eff; + } else + maxr = k; + } + for (int k = 0; k < effective; ++k) + container.add(hardbitmap[k]); + Arrays.fill(hardbitmap, 0); + + } + container.setSizeInBits(range); + } + + /** + * Compute the xor aggregate using a temporary uncompressed bitmap. + * @param bitmaps the source bitmaps + * @param bufsize buffer size used during the computation in 64-bit words + * @return the xor aggregate. + */ + public static EWAHCompressedBitmap32 bufferedxor(final int bufsize, + final EWAHCompressedBitmap32... bitmaps) { + EWAHCompressedBitmap32 answer = new EWAHCompressedBitmap32(); + bufferedxorWithContainer(answer, bufsize, bitmaps); + return answer; + } + + + /** + * Compute the xor aggregate using a temporary uncompressed bitmap. + * + * @param container where the aggregate is written + * @param bufsize buffer size used during the computation in 64-bit words + * @param bitmaps the source bitmaps + */ + public static void bufferedxorWithContainer(final BitmapStorage32 container,final int bufsize, + final EWAHCompressedBitmap32... bitmaps) { + int range = 0; + EWAHCompressedBitmap32[] sbitmaps = bitmaps.clone(); + Arrays.sort(sbitmaps, new Comparator() { + @Override + public int compare(EWAHCompressedBitmap32 a, EWAHCompressedBitmap32 b) { + return b.sizeinbits - a.sizeinbits; + } + }); + + java.util.ArrayList al = new java.util.ArrayList(); + for (EWAHCompressedBitmap32 bitmap : sbitmaps) { + if (bitmap.sizeinbits > range) + range = bitmap.sizeinbits; + al.add(new IteratingBufferedRunningLengthWord32(bitmap)); + } + int[] hardbitmap = new int[bufsize]; + int maxr = al.size(); + while (maxr > 0) { + int effective = 0; + for (int k = 0; k < maxr; ++k) { + if (al.get(k).size() > 0) { + int eff = IteratorAggregation32.inplacexor(hardbitmap, al.get(k)); + if (eff > effective) + effective = eff; + } else + maxr = k; + } + for (int k = 0; k < effective; ++k) + container.add(hardbitmap[k]); + Arrays.fill(hardbitmap, 0); + } + container.setSizeInBits(range); + } + + /** + * Uses a priority queue to compute the or aggregate. + * @param container where we write the result + * @param bitmaps to be aggregated + */ + public static void orToContainer(final BitmapStorage32 container, + final EWAHCompressedBitmap32 ... bitmaps) { + if(bitmaps.length < 2) throw new IllegalArgumentException("We need at least two bitmaps"); + PriorityQueue pq = new PriorityQueue(bitmaps.length, + new Comparator() { + @Override + public int compare(EWAHCompressedBitmap32 a, EWAHCompressedBitmap32 b) { + return a.sizeInBytes() - b.sizeInBytes(); + } + }); + for (EWAHCompressedBitmap32 x : bitmaps) { + pq.add(x); + } + while (pq.size() > 2) { + EWAHCompressedBitmap32 x1 = pq.poll(); + EWAHCompressedBitmap32 x2 = pq.poll(); + pq.add(x1.or(x2)); + } + pq.poll().orToContainer(pq.poll(), container); + } + + + /** + * Uses a priority queue to compute the xor aggregate. + * @param container where we write the result + * @param bitmaps to be aggregated + */ + public static void xorToContainer(final BitmapStorage32 container, + final EWAHCompressedBitmap32 ... bitmaps) { + if(bitmaps.length < 2) throw new IllegalArgumentException("We need at least two bitmaps"); + PriorityQueue pq = new PriorityQueue(bitmaps.length, + new Comparator() { + @Override + public int compare(EWAHCompressedBitmap32 a, EWAHCompressedBitmap32 b) { + return a.sizeInBytes() - b.sizeInBytes(); + } + }); + for (EWAHCompressedBitmap32 x : bitmaps) { + pq.add(x); + } + while (pq.size() > 2) { + EWAHCompressedBitmap32 x1 = pq.poll(); + EWAHCompressedBitmap32 x2 = pq.poll(); + pq.add(x1.xor(x2)); + } + pq.poll().xorToContainer(pq.poll(), container); + } + + /** + * For internal use. Computes the bitwise or of the provided bitmaps and + * stores the result in the container. (This used to be the default.) + * + * @deprecated use EWAHCompressedBitmap32.or instead + * @since 0.4.0 + * @param container where store the result + * @param bitmaps to be aggregated + */ + @Deprecated + public static void legacy_orWithContainer(final BitmapStorage32 container, + final EWAHCompressedBitmap32... bitmaps) { + if (bitmaps.length == 2) { + // should be more efficient + bitmaps[0].orToContainer(bitmaps[1], container); + return; + } + + // Sort the bitmaps in descending order by sizeinbits. We will exhaust the + // sorted bitmaps from right to left. + final EWAHCompressedBitmap32[] sortedBitmaps = bitmaps.clone(); + Arrays.sort(sortedBitmaps, new Comparator() { + @Override + public int compare(EWAHCompressedBitmap32 a, EWAHCompressedBitmap32 b) { + return a.sizeinbits < b.sizeinbits ? 1 + : a.sizeinbits == b.sizeinbits ? 0 : -1; + } + }); + + final IteratingBufferedRunningLengthWord32[] rlws = new IteratingBufferedRunningLengthWord32[bitmaps.length]; + int maxAvailablePos = 0; + for (EWAHCompressedBitmap32 bitmap : sortedBitmaps) { + EWAHIterator32 iterator = bitmap.getEWAHIterator(); + if (iterator.hasNext()) { + rlws[maxAvailablePos++] = new IteratingBufferedRunningLengthWord32( + iterator); + } + } + + if (maxAvailablePos == 0) { // this never happens... + container.setSizeInBits(0); + return; + } + + int maxSize = sortedBitmaps[0].sizeinbits; + + while (true) { + int maxOneRl = 0; + int minZeroRl = Integer.MAX_VALUE; + int minSize = Integer.MAX_VALUE; + int numEmptyRl = 0; + for (int i = 0; i < maxAvailablePos; i++) { + IteratingBufferedRunningLengthWord32 rlw = rlws[i]; + int size = rlw.size(); + if (size == 0) { + maxAvailablePos = i; + break; + } + minSize = Math.min(minSize, size); + + if (rlw.getRunningBit()) { + int rl = rlw.getRunningLength(); + maxOneRl = Math.max(maxOneRl, rl); + minZeroRl = 0; + if (rl == 0 && size > 0) { + numEmptyRl++; + } + } else { + int rl = rlw.getRunningLength(); + minZeroRl = Math.min(minZeroRl, rl); + if (rl == 0 && size > 0) { + numEmptyRl++; + } + } + } + + if (maxAvailablePos == 0) { + break; + } else if (maxAvailablePos == 1) { + // only one bitmap is left so just write the rest of it out + rlws[0].discharge(container); + break; + } + + if (maxOneRl > 0) { + container.addStreamOfEmptyWords(true, maxOneRl); + for (int i = 0; i < maxAvailablePos; i++) { + IteratingBufferedRunningLengthWord32 rlw = rlws[i]; + rlw.discardFirstWords(maxOneRl); + } + } else if (minZeroRl > 0) { + container.addStreamOfEmptyWords(false, minZeroRl); + for (int i = 0; i < maxAvailablePos; i++) { + IteratingBufferedRunningLengthWord32 rlw = rlws[i]; + rlw.discardFirstWords(minZeroRl); + } + } else { + int index = 0; + + if (numEmptyRl == 1) { + // if one rlw has literal words to process and the rest have a run of + // 0's we can write them out here + IteratingBufferedRunningLengthWord32 emptyRl = null; + int minNonEmptyRl = Integer.MAX_VALUE; + for (int i = 0; i < maxAvailablePos; i++) { + IteratingBufferedRunningLengthWord32 rlw = rlws[i]; + int rl = rlw.getRunningLength(); + if (rl == 0) { + assert emptyRl == null; + emptyRl = rlw; + } else { + minNonEmptyRl = Math.min(minNonEmptyRl, rl); + } + } + int wordsToWrite = minNonEmptyRl > minSize ? minSize : minNonEmptyRl; + if (emptyRl != null) + emptyRl.writeLiteralWords(wordsToWrite, container); + index += wordsToWrite; + } + + while (index < minSize) { + int word = 0; + for (int i = 0; i < maxAvailablePos; i++) { + IteratingBufferedRunningLengthWord32 rlw = rlws[i]; + if (rlw.getRunningLength() <= index) { + word |= rlw.getLiteralWordAt(index - rlw.getRunningLength()); + } + } + container.add(word); + index++; + } + for (int i = 0; i < maxAvailablePos; i++) { + IteratingBufferedRunningLengthWord32 rlw = rlws[i]; + rlw.discardFirstWords(minSize); + } + } + } + container.setSizeInBits(maxSize); + } + +} diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah32/IntIteratorImpl32.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah32/IntIteratorImpl32.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah32/IntIteratorImpl32.java 2013-06-11 17:48:37.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah32/IntIteratorImpl32.java 2013-11-12 14:31:20.000000000 +0000 @@ -2,7 +2,7 @@ /* * Copyright 2012, Google Inc. - * Licensed under APL 2.0. + * Licensed under the Apache License, Version 2.0. */ import static com.googlecode.javaewah32.EWAHCompressedBitmap32.wordinbits; @@ -45,10 +45,12 @@ return true; } + @Override public final boolean hasNext() { return this.hasnext; } + @Override public final int next() { final int answer; if (runningHasNext()) { diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah32/IntIteratorOverIteratingRLW32.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah32/IntIteratorOverIteratingRLW32.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah32/IntIteratorOverIteratingRLW32.java 1970-01-01 00:00:00.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah32/IntIteratorOverIteratingRLW32.java 2013-11-12 14:31:20.000000000 +0000 @@ -0,0 +1,91 @@ +package com.googlecode.javaewah32; + +import static com.googlecode.javaewah.EWAHCompressedBitmap.wordinbits; + +import com.googlecode.javaewah.IntIterator; + +/* + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. + */ +/** + * Implementation of an IntIterator over an IteratingRLW. + * + * + */ +public class IntIteratorOverIteratingRLW32 implements IntIterator { + IteratingRLW32 parent; + private int position; + private int runningLength; + private int word; + private int wordPosition; + private int wordLength; + private int literalPosition; + private boolean hasnext; + + /** + * @param p iterator we wish to iterate over + */ + public IntIteratorOverIteratingRLW32(final IteratingRLW32 p) { + this.parent = p; + this.position = 0; + setupForCurrentRunningLengthWord(); + this.hasnext = moveToNext(); + } + + /** + * @return whether we could find another set bit; don't move if there is an unprocessed value + */ + private final boolean moveToNext() { + while (!runningHasNext() && !literalHasNext()) { + if (this.parent.next()) + setupForCurrentRunningLengthWord(); + else return false; + } + return true; + } + + @Override + public boolean hasNext() { + return this.hasnext; + } + + @Override + public final int next() { + final int answer; + if (runningHasNext()) { + answer = this.position++; + } else { + final int bit = Long.numberOfTrailingZeros(this.word); + this.word ^= (1l << bit); + answer = this.literalPosition + bit; + } + this.hasnext = this.moveToNext(); + return answer; + } + + private final void setupForCurrentRunningLengthWord() { + this.runningLength = wordinbits * this.parent.getRunningLength() + + this.position; + + if (!this.parent.getRunningBit()) { + this.position = this.runningLength; + } + this.wordPosition = 0; + this.wordLength = this.parent.getNumberOfLiteralWords(); + } + + private final boolean runningHasNext() { + return this.position < this.runningLength; + } + + private final boolean literalHasNext() { + while (this.word == 0 && this.wordPosition < this.wordLength) { + this.word = this.parent.getLiteralWordAt(this.wordPosition++); + this.literalPosition = this.position; + this.position += wordinbits; + } + return this.word != 0; + } +} + diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah32/IteratingBufferedRunningLengthWord32.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah32/IteratingBufferedRunningLengthWord32.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah32/IteratingBufferedRunningLengthWord32.java 2013-06-11 17:48:37.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah32/IteratingBufferedRunningLengthWord32.java 2013-11-12 14:31:20.000000000 +0000 @@ -3,8 +3,8 @@ /* - * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc. and Veronika Zenz - * Licensed under APL 2.0. + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. */ /** * Mostly for internal use. Similar to BufferedRunningLengthWord32, but automatically @@ -13,7 +13,7 @@ * @since 0.5.0 * @author Daniel Lemire and David McIntosh */ -public final class IteratingBufferedRunningLengthWord32 { +public final class IteratingBufferedRunningLengthWord32 implements IteratingRLW32, Cloneable { /** * Instantiates a new iterating buffered running length word. * @@ -29,8 +29,8 @@ /** * Instantiates a new iterating buffered running length word. + * @param bitmap over which we want to iterate * - * @param iterator iterator */ public IteratingBufferedRunningLengthWord32(final EWAHCompressedBitmap32 bitmap) { this(EWAHIterator32.getEWAHIterator(bitmap)); @@ -42,7 +42,8 @@ * * @param x the x */ - public void discardFirstWords(int x) { + @Override +public void discardFirstWords(int x) { while (x > 0) { if (this.brlw.RunningLength > x) { @@ -107,19 +108,33 @@ pl = max - index; } container.addStreamOfEmptyWords(!getRunningBit(), pl); - discardFirstWords(pl); index += pl; int pd = getNumberOfLiteralWords(); if (pd + index > max) { pd = max - index; } writeNegatedLiteralWords(pd, container); - discardFirstWords(pd); + discardFirstWords(pl+pd); index += pd; } return index; } + /** + * Move to the next RunningLengthWord + * @return whether the move was possible + */ + @Override +public boolean next() { + if (!this.iterator.hasNext()) { + this.brlw.NumberOfLiteralWords = 0; + this.brlw.RunningLength = 0; + return false; + } + this.brlw.reset(this.iterator.next()); + this.literalWordStartPosition = this.iterator.literalWords(); // + this.brlw.literalwordoffset ==0 + return true; + } /** * Write out the remain words, transforming them to zeroes. @@ -147,7 +162,8 @@ * @param index zero based index * @return the literal word */ - public int getLiteralWordAt(int index) { + @Override +public int getLiteralWordAt(int index) { return this.buffer[this.literalWordStartPosition + index]; } @@ -156,7 +172,8 @@ * * @return the number of literal words */ - public int getNumberOfLiteralWords() { + @Override +public int getNumberOfLiteralWords() { return this.brlw.NumberOfLiteralWords; } @@ -165,7 +182,8 @@ * * @return the running bit */ - public boolean getRunningBit() { + @Override +public boolean getRunningBit() { return this.brlw.RunningBit; } @@ -174,7 +192,8 @@ * * @return the running length */ - public int getRunningLength() { + @Override +public int getRunningLength() { return this.brlw.RunningLength; } @@ -183,14 +202,15 @@ * * @return the int */ - public int size() { + @Override +public int size() { return this.brlw.size(); } /** * write the first N literal words to the target bitmap. Does not discard the words or perform iteration. - * @param numWords - * @param container + * @param numWords number of words to be written + * @param container where we write the data */ public void writeLiteralWords(int numWords, BitmapStorage32 container) { container.addStreamOfLiteralWords(this.buffer, this.literalWordStartPosition, numWords); @@ -199,8 +219,8 @@ /** * write the first N literal words (negated) to the target bitmap. Does not discard the words or perform iteration. - * @param numWords - * @param container + * @param numWords number of words to be written + * @param container where we write the data */ public void writeNegatedLiteralWords(int numWords, BitmapStorage32 container) { container.addStreamOfNegatedLiteralWords(this.buffer, this.literalWordStartPosition, numWords); @@ -208,7 +228,7 @@ /** - * For internal use. (One could use the non-static dischard method instead, + * For internal use. (One could use the non-static discharge method instead, * but we expect them to be slower.) * * @param initialWord @@ -234,6 +254,19 @@ runningLengthWord = new BufferedRunningLengthWord32(iterator.next()); } } + + + + @Override +public IteratingBufferedRunningLengthWord32 clone() throws CloneNotSupportedException { + IteratingBufferedRunningLengthWord32 answer = (IteratingBufferedRunningLengthWord32) super.clone(); + answer.brlw = this.brlw.clone(); + answer.buffer = this.buffer; + answer.iterator = this.iterator.clone(); + answer.literalWordStartPosition = this.literalWordStartPosition; + return answer; + } + private BufferedRunningLengthWord32 brlw; private int[] buffer; private int literalWordStartPosition; diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah32/IteratingRLW32.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah32/IteratingRLW32.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah32/IteratingRLW32.java 1970-01-01 00:00:00.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah32/IteratingRLW32.java 2013-11-12 14:31:20.000000000 +0000 @@ -0,0 +1,42 @@ +package com.googlecode.javaewah32; + +/* + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. + */ + +/** + * High-level iterator over a compressed bitmap. + * + */ +public interface IteratingRLW32 { + /** + * @return whether there is more + */ + public boolean next() ; + /** + * @param index where the literal word is + * @return the literal word at the given index. + */ + public int getLiteralWordAt(int index); + /** + * @return the number of literal (non-fill) words + */ + public int getNumberOfLiteralWords() ; + /** + * @return the bit used for the fill bits + */ + public boolean getRunningBit() ; + /** + * @return sum of getRunningLength() and getNumberOfLiteralWords() + */ + public int size() ; + /** + * @return length of the run of fill words + */ + public int getRunningLength() ; + /** + * @param x the number of words to discard + */ + public void discardFirstWords(int x); +} diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah32/IteratorAggregation32.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah32/IteratorAggregation32.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah32/IteratorAggregation32.java 1970-01-01 00:00:00.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah32/IteratorAggregation32.java 2013-11-12 14:31:20.000000000 +0000 @@ -0,0 +1,601 @@ +package com.googlecode.javaewah32; + +import java.util.Arrays; +import java.util.Iterator; +import java.util.LinkedList; + +import com.googlecode.javaewah.CloneableIterator; + + + +/* + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. + */ + +/** + * Set of helper functions to aggregate bitmaps. + * + */ +public class IteratorAggregation32 { + /** + * @param x iterator to negate + * @return negated version of the iterator + */ + public static IteratingRLW32 not(final IteratingRLW32 x) { + return new IteratingRLW32() { + + @Override + public boolean next() { + return x.next(); + } + + @Override + public int getLiteralWordAt(int index) { + return ~x.getLiteralWordAt(index); + } + + @Override + public int getNumberOfLiteralWords() { + return x.getNumberOfLiteralWords(); + } + + @Override + public boolean getRunningBit() { + return ! x.getRunningBit(); + } + + @Override + public int size() { + return x.size(); + } + + @Override + public int getRunningLength() { + return x.getRunningLength(); + } + + @Override + public void discardFirstWords(int y) { + x.discardFirstWords(y); + } + + }; + } + + /** + * Aggregate the iterators using a bitmap buffer. + * + * @param al iterators to aggregate + * @return and aggregate + */ + public static IteratingRLW32 bufferedand(final IteratingRLW32... al) { + return bufferedand (DEFAULTMAXBUFSIZE,al); + } + + /** + * Aggregate the iterators using a bitmap buffer. + * + * @param al iterators to aggregate + * @param bufsize size of the internal buffer used by the iterator in 64-bit words + * @return and aggregate + */ + public static IteratingRLW32 bufferedand(final int bufsize, final IteratingRLW32... al) { + if (al.length == 0) + throw new IllegalArgumentException("Need at least one iterator"); + if (al.length == 1) + return al[0]; + final LinkedList basell = new LinkedList(); + for (IteratingRLW32 i : al) + basell.add(i); + return new BufferedIterator32(new AndIt(basell,bufsize)); + } + + /** + * Aggregate the iterators using a bitmap buffer. + * + * @param al iterators to aggregate + * @return or aggregate + */ + public static IteratingRLW32 bufferedor(final IteratingRLW32... al) { + return bufferedor(DEFAULTMAXBUFSIZE,al); + } + + /** + * Aggregate the iterators using a bitmap buffer. + * + * @param al iterators to aggregate + * @param bufsize size of the internal buffer used by the iterator in 64-bit words + * @return or aggregate + */ + public static IteratingRLW32 bufferedor(final int bufsize, final IteratingRLW32... al) { + if (al.length == 0) + throw new IllegalArgumentException("Need at least one iterator"); + if (al.length == 1) + return al[0]; + + final LinkedList basell = new LinkedList(); + for (IteratingRLW32 i : al) + basell.add(i); + return new BufferedIterator32(new ORIt(basell,bufsize)); + } + + /** + * Aggregate the iterators using a bitmap buffer. + * + * @param al iterators to aggregate + * @return xor aggregate + */ + public static IteratingRLW32 bufferedxor(final IteratingRLW32... al) { + return bufferedxor (DEFAULTMAXBUFSIZE,al); + } + /** + * Aggregate the iterators using a bitmap buffer. + * + * @param al iterators to aggregate + * @param bufsize size of the internal buffer used by the iterator in 64-bit words + * @return xor aggregate + */ + public static IteratingRLW32 bufferedxor(final int bufsize, final IteratingRLW32... al) { + if (al.length == 0) + throw new IllegalArgumentException("Need at least one iterator"); + if (al.length == 1) + return al[0]; + + final LinkedList basell = new LinkedList(); + for (IteratingRLW32 i : al) + basell.add(i); + return new BufferedIterator32(new XORIt(basell,bufsize)); + } + /** + * Write out the content of the iterator, but as if it were all zeros. + * + * @param container + * where we write + * @param i + * the iterator + */ + protected static void dischargeAsEmpty(final BitmapStorage32 container, + final IteratingRLW32 i) { + while (i.size() > 0) { + container.addStreamOfEmptyWords(false, i.size()); + i.next(); + + } + } + + /** + * Write out up to max words, returns how many were written + * @param container target for writes + * @param i source of data + * @param max maximal number of writes + * @return how many written + */ + protected static int discharge(final BitmapStorage32 container, IteratingRLW32 i, int max) { + int counter = 0; + while (i.size() > 0 && counter < max) { + int L1 = i.getRunningLength(); + if (L1 > 0) { + if (L1 + counter > max) + L1 = max - counter; + container.addStreamOfEmptyWords(i.getRunningBit(), L1); + counter += L1; + } + int L = i.getNumberOfLiteralWords(); + if(L + counter > max) L = max - counter; + for (int k = 0; k < L; ++k) { + container.add(i.getLiteralWordAt(k)); + } + counter += L; + i.discardFirstWords(L+L1); + } + return counter; + } + + /** + * Write out up to max negated words, returns how many were written + * @param container target for writes + * @param i source of data + * @param max maximal number of writes + * @return how many written + */ + protected static int dischargeNegated(final BitmapStorage32 container, IteratingRLW32 i, int max) { + int counter = 0; + while (i.size() > 0 && counter < max) { + int L1 = i.getRunningLength(); + if (L1 > 0) { + if (L1 + counter > max) + L1 = max - counter; + container.addStreamOfEmptyWords(i.getRunningBit(), L1); + counter += L1; + } + int L = i.getNumberOfLiteralWords(); + if(L + counter > max) L = max - counter; + for (int k = 0; k < L; ++k) { + container.add(i.getLiteralWordAt(k)); + } + counter += L; + i.discardFirstWords(L+L1); + } + return counter; + } + + static void andToContainer(final BitmapStorage32 container, + int desiredrlwcount, final IteratingRLW32 rlwi, IteratingRLW32 rlwj) { + while ((rlwi.size()>0) && (rlwj.size()>0) && (desiredrlwcount-- >0) ) { + while ((rlwi.getRunningLength() > 0) || (rlwj.getRunningLength() > 0)) { + final boolean i_is_prey = rlwi.getRunningLength() < rlwj + .getRunningLength(); + final IteratingRLW32 prey = i_is_prey ? rlwi : rlwj; + final IteratingRLW32 predator = i_is_prey ? rlwj + : rlwi; + if (predator.getRunningBit() == false) { + container.addStreamOfEmptyWords(false, predator.getRunningLength()); + prey.discardFirstWords(predator.getRunningLength()); + predator.discardFirstWords(predator.getRunningLength()); + } else { + final int index = discharge(container, prey, predator.getRunningLength()); + container.addStreamOfEmptyWords(false, predator.getRunningLength() + - index); + predator.discardFirstWords(predator.getRunningLength()); + } + } + final int nbre_literal = Math.min(rlwi.getNumberOfLiteralWords(), + rlwj.getNumberOfLiteralWords()); + if (nbre_literal > 0) { + desiredrlwcount -= nbre_literal; + for (int k = 0; k < nbre_literal; ++k) + container.add(rlwi.getLiteralWordAt(k) & rlwj.getLiteralWordAt(k)); + rlwi.discardFirstWords(nbre_literal); + rlwj.discardFirstWords(nbre_literal); + } + } + } + + static void andToContainer(final BitmapStorage32 container, + final IteratingRLW32 rlwi, IteratingRLW32 rlwj) { + while ((rlwi.size()>0) && (rlwj.size()>0) ) { + while ((rlwi.getRunningLength() > 0) || (rlwj.getRunningLength() > 0)) { + final boolean i_is_prey = rlwi.getRunningLength() < rlwj + .getRunningLength(); + final IteratingRLW32 prey = i_is_prey ? rlwi : rlwj; + final IteratingRLW32 predator = i_is_prey ? rlwj + : rlwi; + if (predator.getRunningBit() == false) { + container.addStreamOfEmptyWords(false, predator.getRunningLength()); + prey.discardFirstWords(predator.getRunningLength()); + predator.discardFirstWords(predator.getRunningLength()); + } else { + final int index = discharge(container, prey, predator.getRunningLength()); + container.addStreamOfEmptyWords(false, predator.getRunningLength() + - index); + predator.discardFirstWords(predator.getRunningLength()); + } + } + final int nbre_literal = Math.min(rlwi.getNumberOfLiteralWords(), + rlwj.getNumberOfLiteralWords()); + if (nbre_literal > 0) { + for (int k = 0; k < nbre_literal; ++k) + container.add(rlwi.getLiteralWordAt(k) & rlwj.getLiteralWordAt(k)); + rlwi.discardFirstWords(nbre_literal); + rlwj.discardFirstWords(nbre_literal); + } + } + } + + + /** + * Compute the first few words of the XOR aggregate between two iterators. + * + * @param container where to write + * @param desiredrlwcount number of words to be written (max) + * @param rlwi first iterator to aggregate + * @param rlwj second iterator to aggregate + */ + public static void xorToContainer(final BitmapStorage32 container, + int desiredrlwcount, final IteratingRLW32 rlwi, IteratingRLW32 rlwj) { + while ((rlwi.size()>0) && (rlwj.size()>0) && (desiredrlwcount-- >0) ) { + while ((rlwi.getRunningLength() > 0) || (rlwj.getRunningLength() > 0)) { + final boolean i_is_prey = rlwi.getRunningLength() < rlwj + .getRunningLength(); + final IteratingRLW32 prey = i_is_prey ? rlwi : rlwj; + final IteratingRLW32 predator = i_is_prey ? rlwj + : rlwi; + if (predator.getRunningBit() == false) { + int index = discharge(container, prey, predator.getRunningLength()); + container.addStreamOfEmptyWords(false, predator.getRunningLength() + - index); + predator.discardFirstWords(predator.getRunningLength()); + } else { + int index = dischargeNegated(container, prey, predator.getRunningLength()); + container.addStreamOfEmptyWords(true, predator.getRunningLength() + - index); + predator.discardFirstWords(predator.getRunningLength()); + } + } + final int nbre_literal = Math.min(rlwi.getNumberOfLiteralWords(), + rlwj.getNumberOfLiteralWords()); + if (nbre_literal > 0) { + desiredrlwcount -= nbre_literal; + for (int k = 0; k < nbre_literal; ++k) + container.add(rlwi.getLiteralWordAt(k) ^ rlwj.getLiteralWordAt(k)); + rlwi.discardFirstWords(nbre_literal); + rlwj.discardFirstWords(nbre_literal); + } + } + } + + protected static int inplaceor(int[] bitmap, + IteratingRLW32 i) { + int pos = 0; + int s; + while ((s = i.size()) > 0) { + if (pos + s < bitmap.length) { + final int L = i.getRunningLength(); + if (i.getRunningBit()) + java.util.Arrays.fill(bitmap, pos, pos + L, ~0); + pos += L; + final int LR = i.getNumberOfLiteralWords(); + for (int k = 0; k < LR; ++k) + bitmap[pos++] |= i.getLiteralWordAt(k); + if (!i.next()) { + return pos; + } + } else { + int howmany = bitmap.length - pos; + int L = i.getRunningLength(); + if (pos + L > bitmap.length) { + if (i.getRunningBit()) { + java.util.Arrays.fill(bitmap, pos, bitmap.length, ~0); + } + i.discardFirstWords(howmany); + return bitmap.length; + } + if (i.getRunningBit()) + java.util.Arrays.fill(bitmap, pos, pos + L, ~0); + pos += L; + for (int k = 0; pos < bitmap.length; ++k) + bitmap[pos++] |= i.getLiteralWordAt(k); + i.discardFirstWords(howmany); + return pos; + } + } + return pos; + } + + + protected static int inplacexor(int[] bitmap, + IteratingRLW32 i) { + int pos = 0; + int s; + while ((s = i.size()) > 0) { + if (pos + s < bitmap.length) { + final int L = i.getRunningLength(); + if (i.getRunningBit()) { + for(int k = pos ; k < pos + L; ++k) + bitmap[k] = ~bitmap[k]; + } + pos += L; + final int LR = i.getNumberOfLiteralWords(); + for (int k = 0; k < LR; ++k) + bitmap[pos++] ^= i.getLiteralWordAt(k); + if (!i.next()) { + return pos; + } + } else { + int howmany = bitmap.length - pos; + int L = i.getRunningLength(); + if (pos + L > bitmap.length) { + if (i.getRunningBit()) { + for(int k = pos ; k < bitmap.length; ++k) + bitmap[k] = ~bitmap[k]; + } + i.discardFirstWords(howmany); + return bitmap.length; + } + if (i.getRunningBit()) + for(int k = pos ; k < pos + L; ++k) + bitmap[k] = ~bitmap[k]; + pos += L; + for (int k = 0; pos < bitmap.length; ++k) + bitmap[pos++] ^= i.getLiteralWordAt(k); + i.discardFirstWords(howmany); + return pos; + } + } + return pos; + } + protected static int inplaceand(int[] bitmap, + IteratingRLW32 i) { + int pos = 0; + int s; + while ((s = i.size()) > 0) { + if (pos + s < bitmap.length) { + final int L = i.getRunningLength(); + if (!i.getRunningBit()) { + for(int k = pos ; k < pos + L; ++k) + bitmap[k] = 0; + } + pos += L; + final int LR = i.getNumberOfLiteralWords(); + for (int k = 0; k < LR; ++k) + bitmap[pos++] &= i.getLiteralWordAt(k); + if (!i.next()) { + return pos; + } + } else { + int howmany = bitmap.length - pos; + int L = i.getRunningLength(); + if (pos + L > bitmap.length) { + if (!i.getRunningBit()) { + for(int k = pos ; k < bitmap.length; ++k) + bitmap[k] = 0; + } + i.discardFirstWords(howmany); + return bitmap.length; + } + if (!i.getRunningBit()) + for(int k = pos ; k < pos + L; ++k) + bitmap[k] = 0; + pos += L; + for (int k = 0; pos < bitmap.length; ++k) + bitmap[pos++] &= i.getLiteralWordAt(k); + i.discardFirstWords(howmany); + return pos; + } + } + return pos; + } + + /** + * An optimization option. Larger values may improve speed, but at + * the expense of memory. + */ + public final static int DEFAULTMAXBUFSIZE = 65536; + + +} + + +class ORIt implements CloneableIterator { + EWAHCompressedBitmap32 buffer = new EWAHCompressedBitmap32(); + int[] hardbitmap; + LinkedList ll; + + ORIt(LinkedList basell, final int bufsize) { + this.ll = basell; + this.hardbitmap = new int[bufsize]; + } + + @Override + public XORIt clone() throws CloneNotSupportedException { + XORIt answer = (XORIt) super.clone(); + answer.buffer = this.buffer.clone(); + answer.hardbitmap = this.hardbitmap.clone(); + answer.ll = (LinkedList) this.ll.clone(); + return answer; + } + + @Override + public boolean hasNext() { + return !this.ll.isEmpty(); + } + + @Override + public EWAHIterator32 next() { + this.buffer.clear(); + int effective = 0; + Iterator i = this.ll.iterator(); + while (i.hasNext()) { + IteratingRLW32 rlw = i.next(); + if (rlw.size() > 0) { + int eff = IteratorAggregation32.inplaceor(this.hardbitmap, rlw); + if (eff > effective) + effective = eff; + } else + i.remove(); + } + for (int k = 0; k < effective; ++k) + this.buffer.add(this.hardbitmap[k]); + Arrays.fill(this.hardbitmap, 0); + return this.buffer.getEWAHIterator(); + } +} + +class XORIt implements CloneableIterator { + EWAHCompressedBitmap32 buffer = new EWAHCompressedBitmap32(); + int[] hardbitmap; + LinkedList ll; + + XORIt(LinkedList basell, final int bufsize) { + this.ll = basell; + this.hardbitmap = new int[bufsize]; + + } + + @Override + public XORIt clone() throws CloneNotSupportedException { + XORIt answer = (XORIt) super.clone(); + answer.buffer = this.buffer.clone(); + answer.hardbitmap = this.hardbitmap.clone(); + answer.ll = (LinkedList) this.ll.clone(); + return answer; + } + + @Override + public boolean hasNext() { + return !this.ll.isEmpty(); + } + + @Override + public EWAHIterator32 next() { + this.buffer.clear(); + int effective = 0; + Iterator i = this.ll.iterator(); + while (i.hasNext()) { + IteratingRLW32 rlw = i.next(); + if (rlw.size() > 0) { + int eff = IteratorAggregation32.inplacexor(this.hardbitmap, rlw); + if (eff > effective) + effective = eff; + } else + i.remove(); + } + for (int k = 0; k < effective; ++k) + this.buffer.add(this.hardbitmap[k]); + Arrays.fill(this.hardbitmap, 0); + return this.buffer.getEWAHIterator(); + } +} + +class AndIt implements CloneableIterator { + EWAHCompressedBitmap32 buffer = new EWAHCompressedBitmap32(); + LinkedList ll; + int buffersize; + + public AndIt(LinkedList basell, final int bufsize) { + this.ll = basell; + this.buffersize = bufsize; + } + + @Override + public boolean hasNext() { + return !this.ll.isEmpty(); + } + + @Override + public AndIt clone() throws CloneNotSupportedException { + AndIt answer = (AndIt) super.clone(); + answer.buffer = this.buffer.clone(); + answer.ll = (LinkedList) this.ll.clone(); + return answer; + } + + @Override + public EWAHIterator32 next() { + this.buffer.clear(); + IteratorAggregation32.andToContainer(this.buffer, this.buffersize * this.ll.size(), + this.ll.get(0), this.ll.get(1)); + if (this.ll.size() > 2) { + Iterator i = this.ll.iterator(); + i.next(); + i.next(); + EWAHCompressedBitmap32 tmpbuffer = new EWAHCompressedBitmap32(); + while (i.hasNext() && this.buffer.sizeInBytes() > 0) { + IteratorAggregation32.andToContainer(tmpbuffer, + this.buffer.getIteratingRLW(), i.next()); + this.buffer.swap(tmpbuffer); + tmpbuffer.clear(); + } + } + Iterator i = this.ll.iterator(); + while(i.hasNext()) { + if(i.next().size() == 0) { + this.ll.clear(); + break; + } + } + return this.buffer.getEWAHIterator(); + } + +} \ No newline at end of file diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah32/IteratorUtil32.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah32/IteratorUtil32.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah32/IteratorUtil32.java 1970-01-01 00:00:00.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah32/IteratorUtil32.java 2013-11-12 14:31:20.000000000 +0000 @@ -0,0 +1,135 @@ +package com.googlecode.javaewah32; + +import java.util.Iterator; + +import com.googlecode.javaewah.IntIterator; + + +/* + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. + */ +/** + * Convenience functions for working over iterators + * + */ +public class IteratorUtil32 { + + /** + * @param i iterator we wish to iterate over + * @return an iterator over the set bits corresponding to the iterator + */ + public static IntIterator toSetBitsIntIterator(final IteratingRLW32 i) { + return new IntIteratorOverIteratingRLW32(i); + } + + /** + * @param i iterator we wish to iterate over + * @return an iterator over the set bits corresponding to the iterator + */ + public static Iterator toSetBitsIterator(final IteratingRLW32 i) { + return new Iterator() { + @Override + public boolean hasNext() { + return this.under.hasNext(); + } + + @Override + public Integer next() { + return new Integer(this.under.next()); + } + + @Override + public void remove() { + } + + final private IntIterator under = toSetBitsIntIterator(i); + }; + + } + + /** + * Turn an iterator into a bitmap + * @param i iterator we wish to materialize + * @param c where we write + */ + public static void materialize(final IteratingRLW32 i, final BitmapStorage32 c) { + while (true) { + if (i.getRunningLength() > 0) { + c.addStreamOfEmptyWords(i.getRunningBit(), i.getRunningLength()); + } + for (int k = 0; k < i.getNumberOfLiteralWords(); ++k) + c.add(i.getLiteralWordAt(k)); + if (!i.next()) + break; + } + } + + /** + * @param i iterator we wish to iterate over + * @return the cardinality (number of set bits) corresponding to the iterator + */ + public static int cardinality(final IteratingRLW32 i) { + int answer = 0; + while (true) { + if(i.getRunningBit()) answer += i.getRunningLength() * EWAHCompressedBitmap32.wordinbits; + for (int k = 0; k < i.getNumberOfLiteralWords(); ++k) + answer += Long.bitCount(i.getLiteralWordAt(k)); + if(!i.next()) break; + } + return answer; + } + + /** + * + * @param x set of bitmaps we wish to iterate over + * @return an array of iterators corresponding to the array of bitmaps + */ + public static IteratingRLW32[] toIterators(final EWAHCompressedBitmap32... x) { + IteratingRLW32[] X = new IteratingRLW32[x.length]; + for (int k = 0; k < X.length; ++k) { + X[k] = new IteratingBufferedRunningLengthWord32(x[k]); + } + return X; + } + /** + * Turn an iterator into a bitmap + * + * @param i iterator we wish to materialize + * @param c where we write + * @param Max maximum number of words to materialize + * @return how many words were actually materialized + */ + public static long materialize(final IteratingRLW32 i, final BitmapStorage32 c, int Max) { + final int origMax = Max; + while (true) { + if (i.getRunningLength() > 0) { + int L = i.getRunningLength(); + if(L > Max) L = Max; + c.addStreamOfEmptyWords(i.getRunningBit(), L); + Max -= L; + } + long L = i.getNumberOfLiteralWords(); + for (int k = 0; k < L; ++k) + c.add(i.getLiteralWordAt(k)); + if(Max>0) { + if (!i.next()) + break; + } + else break; + } + return origMax - Max; + } + /** + * Turn an iterator into a bitmap + * + * @param i iterator we wish to materialize + * @return materialized version of the iterator + */ + public static EWAHCompressedBitmap32 materialize(final IteratingRLW32 i) { + EWAHCompressedBitmap32 ewah = new EWAHCompressedBitmap32(); + materialize(i, ewah); + return ewah; + } + +} diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah32/NonEmptyVirtualStorage32.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah32/NonEmptyVirtualStorage32.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah32/NonEmptyVirtualStorage32.java 2013-06-11 17:48:37.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah32/NonEmptyVirtualStorage32.java 2013-11-12 14:31:20.000000000 +0000 @@ -3,8 +3,8 @@ /* - * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc. and Veronika Zenz - * Licensed under APL 2.0. + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. */ /** * This is a BitmapStorage that can be used to determine quickly @@ -37,39 +37,39 @@ /** * If the word to be added is non-zero, a NonEmptyException exception is thrown. - * @see com.googlecode.javaewah.BitmapStorage#add(int) */ - public void add(int newdata) { + @Override +public void add(int newdata) { if(newdata!=0) throw nonEmptyException; } /** - * throws a NonEmptyException exception when number > 0 + * throws a NonEmptyException exception when number is greater than 0 * - * @see com.googlecode.javaewah.BitmapStorage#addStreamOfLiteralWords(int[], int, int) */ - public void addStreamOfLiteralWords(int[] data, int start, int number) { + @Override +public void addStreamOfLiteralWords(int[] data, int start, int number) { if (number > 0){ throw nonEmptyException; } } /** - * If the boolean value is true and number>0, then it throws a NonEmptyException exception, + * If the boolean value is true and number is greater than 0, then it throws a NonEmptyException exception, * otherwise, nothing happens. * - * @see com.googlecode.javaewah.BitmapStorage#addStreamOfEmptyWords(boolean, int) */ - public void addStreamOfEmptyWords(boolean v, int number) { + @Override +public void addStreamOfEmptyWords(boolean v, int number) { if(v && (number>0)) throw nonEmptyException; } /** - * throws a NonEmptyException exception when number > 0 + * throws a NonEmptyException exception when number is greater than 0 * - * @see com.googlecode.javaewah.BitmapStorage#addStreamOfNegatedLiteralWords(int[], int, int) */ - public void addStreamOfNegatedLiteralWords(int[] data, int start, int number) { + @Override +public void addStreamOfNegatedLiteralWords(int[] data, int start, int number) { if (number > 0){ throw nonEmptyException; } @@ -80,7 +80,8 @@ * * @see com.googlecode.javaewah.BitmapStorage#setSizeInBits(int) */ - public void setSizeInBits(int bits) { + @Override +public void setSizeInBits(int bits) { } } diff -Nru libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah32/RunningLengthWord32.java libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah32/RunningLengthWord32.java --- libjavaewah-java-0.6.12/src/main/java/com/googlecode/javaewah32/RunningLengthWord32.java 2013-06-11 17:48:37.000000000 +0000 +++ libjavaewah-java-0.7.9/src/main/java/com/googlecode/javaewah32/RunningLengthWord32.java 2013-11-12 14:31:20.000000000 +0000 @@ -1,135 +1,152 @@ package com.googlecode.javaewah32; - /* - * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc. and Veronika Zenz - * Licensed under APL 2.0. + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. */ /** * Mostly for internal use. - * + * * @since 0.5.0 * @author Daniel Lemire */ -public final class RunningLengthWord32 { +public final class RunningLengthWord32 implements Cloneable { + + /** + * Instantiates a new running length word. + * + * @param a + * an array of 32-bit words + * @param p + * position in the array where the running length word is + * located. + */ + RunningLengthWord32(final EWAHCompressedBitmap32 a, final int p) { + this.parent = a; + this.position = p; + } + + /** + * Gets the number of literal words. + * + * @return the number of literal words + */ + public int getNumberOfLiteralWords() { + return (this.parent.buffer[this.position] >>> (1 + runninglengthbits)); + } + + /** + * Gets the running bit. + * + * @return the running bit + */ + public boolean getRunningBit() { + return (this.parent.buffer[this.position] & 1) != 0; + } + + /** + * Gets the running length. + * + * @return the running length + */ + public int getRunningLength() { + return (this.parent.buffer[this.position] >>> 1) + & largestrunninglengthcount; + } + + /** + * Sets the number of literal words. + * + * @param number + * the new number of literal words + */ + public void setNumberOfLiteralWords(final int number) { + this.parent.buffer[this.position] |= notrunninglengthplusrunningbit; + this.parent.buffer[this.position] &= (number << (runninglengthbits + 1)) + | runninglengthplusrunningbit; + } + + /** + * Sets the running bit. + * + * @param b + * the new running bit + */ + public void setRunningBit(final boolean b) { + if (b) + this.parent.buffer[this.position] |= 1; + else + this.parent.buffer[this.position] &= ~1; + } + + /** + * Sets the running length. + * + * @param number + * the new running length + */ + public void setRunningLength(final int number) { + this.parent.buffer[this.position] |= shiftedlargestrunninglengthcount; + this.parent.buffer[this.position] &= (number << 1) + | notshiftedlargestrunninglengthcount; + } + + /** + * Return the size in uncompressed words represented by this running + * length word. + * + * @return the int + */ + public int size() { + return getRunningLength() + getNumberOfLiteralWords(); + } + + /* + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return "running bit = " + getRunningBit() + + " running length = " + getRunningLength() + + " number of lit. words " + getNumberOfLiteralWords(); + } + + @Override + public RunningLengthWord32 clone() throws CloneNotSupportedException { + RunningLengthWord32 answer; + answer = (RunningLengthWord32) super.clone(); + answer.parent = this.parent; + answer.position = this.position; + return answer; + } + + /** The array of words. */ + public EWAHCompressedBitmap32 parent; + + /** The position in array. */ + public int position; + + /** + * number of bits dedicated to marking of the running length of clean + * words + */ + public static final int runninglengthbits = 16; + + private static final int literalbits = 32 - 1 - runninglengthbits; + + /** largest number of literal words in a run. */ + public static final int largestliteralcount = (1 << literalbits) - 1; + + /** largest number of clean words in a run */ + public static final int largestrunninglengthcount = (1 << runninglengthbits) - 1; + + private static final int runninglengthplusrunningbit = (1 << (runninglengthbits + 1)) - 1; + + private static final int shiftedlargestrunninglengthcount = largestrunninglengthcount << 1; + + private static final int notrunninglengthplusrunningbit = ~runninglengthplusrunningbit; + + private static final int notshiftedlargestrunninglengthcount = ~shiftedlargestrunninglengthcount; - /** - * Instantiates a new running length word. - * - * @param a an array of 32-bit words - * @param p position in the array where the running length word is located. - */ - RunningLengthWord32(final int[] a, final int p) { - this.array = a; - this.position = p; - } - - /** - * Gets the number of literal words. - * - * @return the number of literal words - */ - public int getNumberOfLiteralWords() { - return (this.array[this.position] >>> (1 + runninglengthbits)); - } - - /** - * Gets the running bit. - * - * @return the running bit - */ - public boolean getRunningBit() { - return (this.array[this.position] & 1) != 0; - } - - /** - * Gets the running length. - * - * @return the running length - */ - public int getRunningLength() { - return (this.array[this.position] >>> 1) & largestrunninglengthcount; - } - - /** - * Sets the number of literal words. - * - * @param number the new number of literal words - */ - public void setNumberOfLiteralWords(final int number) { - this.array[this.position] |= notrunninglengthplusrunningbit; - this.array[this.position] &= (number << (runninglengthbits + 1)) - | runninglengthplusrunningbit; - } - - /** - * Sets the running bit. - * - * @param b the new running bit - */ - public void setRunningBit(final boolean b) { - if (b) - this.array[this.position] |= 1; - else - this.array[this.position] &= ~1; - } - - /** - * Sets the running length. - * - * @param number the new running length - */ - public void setRunningLength(final int number) { - this.array[this.position] |= shiftedlargestrunninglengthcount; - this.array[this.position] &= (number << 1) - | notshiftedlargestrunninglengthcount; - } - - /** - * Return the size in uncompressed words represented by - * this running length word. - * - * @return the int - */ - public int size() { - return getRunningLength() + getNumberOfLiteralWords(); - } - - /* - * @see java.lang.Object#toString() - */ - @Override - public String toString() { - return "running bit = " + getRunningBit() + " running length = " - + getRunningLength() + " number of lit. words " - + getNumberOfLiteralWords(); - } - - - /** The array of words. */ - public int[] array; - - /** The position in array. */ - public int position; - - /** number of bits dedicated to marking of the running length of clean words */ - public static final int runninglengthbits = 16; - - private static final int literalbits = 32 - 1 - runninglengthbits; - - /** largest number of literal words in a run. */ - public static final int largestliteralcount = (1 << literalbits) - 1; - - /** largest number of clean words in a run */ - public static final int largestrunninglengthcount = (1 << runninglengthbits) - 1; - - private static final int runninglengthplusrunningbit = (1 << (runninglengthbits + 1)) - 1; - - private static final int shiftedlargestrunninglengthcount = largestrunninglengthcount << 1; - - private static final int notrunninglengthplusrunningbit = ~runninglengthplusrunningbit; - - private static final int notshiftedlargestrunninglengthcount = ~shiftedlargestrunninglengthcount; - } \ No newline at end of file diff -Nru libjavaewah-java-0.6.12/src/test/java/com/googlecode/javaewah/EWAHCompressedBitmapTest.java libjavaewah-java-0.7.9/src/test/java/com/googlecode/javaewah/EWAHCompressedBitmapTest.java --- libjavaewah-java-0.6.12/src/test/java/com/googlecode/javaewah/EWAHCompressedBitmapTest.java 2013-06-11 17:48:37.000000000 +0000 +++ libjavaewah-java-0.7.9/src/test/java/com/googlecode/javaewah/EWAHCompressedBitmapTest.java 2013-11-12 14:31:20.000000000 +0000 @@ -1,902 +1,1016 @@ package com.googlecode.javaewah; - /* - * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc. and Veronika Zenz - * Licensed under APL 2.0. + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. */ import org.junit.Test; import java.util.*; import java.io.*; - - - import junit.framework.Assert; /** - * This class is used for unit testing. + * This class is used for basic unit testing. */ +@SuppressWarnings("javadoc") public class EWAHCompressedBitmapTest { - - @Test - public void testDebugSetSizeInBitsTest() { - System.out.println("testing DebugSetSizeInBits"); - EWAHCompressedBitmap b = new EWAHCompressedBitmap(); - - b.set(4); - - b.setSizeInBits(6, true); - - List positions = b.getPositions(); - - Assert.assertEquals(2, positions.size()); - Assert.assertEquals(Integer.valueOf(4), positions.get(0)); - Assert.assertEquals(Integer.valueOf(5), positions.get(1)); - - Iterator iterator = b.iterator(); - Assert.assertTrue(iterator.hasNext()); - Assert.assertEquals(Integer.valueOf(4), iterator.next()); - Assert.assertTrue(iterator.hasNext()); - Assert.assertEquals(Integer.valueOf(5), iterator.next()); - Assert.assertFalse(iterator.hasNext()); - - IntIterator intIterator = b.intIterator(); - Assert.assertTrue(intIterator.hasNext()); - Assert.assertEquals(4, intIterator.next()); - Assert.assertTrue(intIterator.hasNext()); - Assert.assertEquals(5, intIterator.next()); - Assert.assertFalse(intIterator.hasNext()); - - } - - /** - * Created: 2/4/11 6:03 PM By: Arnon Moscona. - */ - @Test - public void EwahIteratorProblem() { - System.out.println("testing ArnonMoscona"); - EWAHCompressedBitmap bitmap = new EWAHCompressedBitmap(); - for (int i = 9434560; i <= 9435159; i++) { - bitmap.set(i); - } - IntIterator iterator = bitmap.intIterator(); - List v = bitmap.getPositions(); - int[] array = bitmap.toArray(); - for (int k = 0; k < v.size(); ++k) { - Assert.assertTrue(array[k]== v.get(k).intValue()); - Assert.assertTrue(iterator.hasNext()); - final int ival = iterator.next(); - final int vval = v.get(k).intValue(); - Assert.assertTrue(ival == vval); - } - Assert.assertTrue(!iterator.hasNext()); - // - for (int k = 2; k <= 1024; k *= 2) { - int[] bitsToSet = createSortedIntArrayOfBitsToSet(k,434455 + 5*k); - EWAHCompressedBitmap ewah = new EWAHCompressedBitmap(); - for (int i : bitsToSet) { - ewah.set(i); - } - equal(ewah.iterator(), bitsToSet); - } - } - - /** - * Test submitted by Gregory Ssi-Yan-Kai - */ - @Test - public void SsiYanKaiTest() { - System.out.println("testing SsiYanKaiTest"); - EWAHCompressedBitmap a = EWAHCompressedBitmap.bitmapOf(39935, 39936, 39937, 39938, 39939, 39940, 39941, 39942, 39943, 39944, 39945, 39946, 39947, 39948, 39949, 39950, 39951, 39952, 39953, 39954, 39955, 39956, 39957, 39958, 39959, 39960, 39961, 39962, 39963, 39964, 39965, 39966, 39967, 39968, 39969, 39970, 39971, 39972, 39973, 39974, 39975, 39976, 39977, 39978, 39979, 39980, 39981, 39982, 39983, 39984, 39985, 39986, 39987, 39988, 39989, 39990, 39991, 39992, 39993, 39994, 39995, 39996, 39997, 39998, 39999, 40000, 40001, 40002, 40003, 40004, 40005, 40006, 40007, 40008, 40009, 40010, 40011, 40012, 40013, 40014, 40015, 40016, 40017, 40018, 40019, 40020, 40021, 40022, 40023, 40024, 40025, 40026, 40027, 40028, 40029, 40030, 40031, 40032, 40033, 40034, 40035, 40036, 40037, 40038, 40039, 40040, 40041, 40042, 40043, 40044, 40045, 40046, 40047, 40048, 40049, 40050, 40051, 40052, 40053, 40054, 40055, 40056, 40057, 40058, 40059, 40060, 40061, 40062, 40063, 40064, 40065, 40066, 40067, 40068, 40069, 40070, 40071, 40072, 40073, 40074, 40075, 40076, 40077, 40078, 40079, 40080, 40081, 40082, 40083, 40084, 40085, 40086, 40087, 40088, 40089, 40090, 40091, 40092, 40093, 40094, 40095, 40096, 40097, 40098, 40099, 40100); - EWAHCompressedBitmap b = EWAHCompressedBitmap.bitmapOf(39935, 39936, 39937, 39938, 39939, 39940, 39941, 39942, 39943, 39944, 39945, 39946, 39947, 39948, 39949, 39950, 39951, 39952, 39953, 39954, 39955, 39956, 39957, 39958, 39959, 39960, 39961, 39962, 39963, 39964, 39965, 39966, 39967, 39968, 39969, 39970, 39971, 39972, 39973, 39974, 39975, 39976, 39977, 39978, 39979, 39980, 39981, 39982, 39983, 39984, 39985, 39986, 39987, 39988, 39989, 39990, 39991, 39992, 39993, 39994, 39995, 39996, 39997, 39998, 39999, 270000); - LinkedHashSet aPositions = new LinkedHashSet(a.getPositions()); - int intersection = 0; - EWAHCompressedBitmap inter = new EWAHCompressedBitmap(); - LinkedHashSet bPositions = new LinkedHashSet(b.getPositions()); - for (Integer integer : bPositions) { - if (aPositions.contains(integer)) { - inter.set(integer.intValue()); - ++intersection; - } - } - EWAHCompressedBitmap and2 = a.and(b); - if(!and2.equals(inter)) throw new RuntimeException("intersections don't match"); - if(intersection != and2.cardinality()) throw new RuntimeException("cardinalities don't match"); - } - - /** - * Test inspired by William Habermaas. - */ - @Test - public void habermaasTest() { - System.out.println("testing habermaasTest"); - BitSet bitsetaa = new BitSet(); - EWAHCompressedBitmap aa = new EWAHCompressedBitmap(); - int[] val = { 55400, 1000000, 1000128 }; - for (int k = 0; k < val.length; ++k) { - aa.set(val[k]); - bitsetaa.set(val[k]); - } - equal(aa, bitsetaa); - BitSet bitsetab = new BitSet(); - EWAHCompressedBitmap ab = new EWAHCompressedBitmap(); - for (int i = 4096; i < (4096 + 5); i++) { - ab.set(i); - bitsetab.set(i); - } - ab.set(99000); - bitsetab.set(99000); - ab.set(1000130); - bitsetab.set(1000130); - equal(ab, bitsetab); - EWAHCompressedBitmap bb = aa.or(ab); - EWAHCompressedBitmap bbAnd = aa.and(ab); - try { - EWAHCompressedBitmap abnot = (EWAHCompressedBitmap)ab.clone(); - abnot.not(); - EWAHCompressedBitmap bbAnd2 = aa.andNot(abnot); - assertEquals(bbAnd2,bbAnd); - } catch (CloneNotSupportedException e) { - e.printStackTrace(); - } - BitSet bitsetbb = (BitSet) bitsetaa.clone(); - bitsetbb.or(bitsetab); - BitSet bitsetbbAnd = (BitSet) bitsetaa.clone(); - bitsetbbAnd.and(bitsetab); - equal(bbAnd, bitsetbbAnd); - equal(bb, bitsetbb); - } - - @Test - public void testAndResultAppend() - { - System.out.println("testing AndResultAppend"); - EWAHCompressedBitmap bitmap1 = new EWAHCompressedBitmap(); - bitmap1.set(35); - EWAHCompressedBitmap bitmap2 = new EWAHCompressedBitmap(); - bitmap2.set(35); - bitmap2.set(130); - - EWAHCompressedBitmap resultBitmap = bitmap1.and(bitmap2); - resultBitmap.set(131); - - bitmap1.set(131); - assertEquals(bitmap1,resultBitmap); - } - - /** - * Test cardinality. - */ - @Test - public void testCardinality() { - System.out.println("testing EWAH cardinality"); - EWAHCompressedBitmap bitmap = new EWAHCompressedBitmap(); - bitmap.set(Integer.MAX_VALUE - 64); - // System.out.format("Total Items %d\n", bitmap.cardinality()); - Assert.assertTrue(bitmap.cardinality() == 1); - } - - /** - * Test clear function - */ - @Test - public void testClear() { - System.out.println("testing Clear"); - EWAHCompressedBitmap bitmap = new EWAHCompressedBitmap(); - bitmap.set(5); - bitmap.clear(); - bitmap.set(7); - Assert.assertTrue(1 == bitmap.cardinality()); - Assert.assertTrue(1 == bitmap.getPositions().size()); - Assert.assertTrue(1 == bitmap.toArray().length); - Assert.assertTrue(7 == bitmap.getPositions().get(0).intValue()); - Assert.assertTrue(7 == bitmap.toArray()[0]); - bitmap.clear(); - bitmap.set( 5000 ); - Assert.assertTrue(1 == bitmap.cardinality()); - Assert.assertTrue(1 == bitmap.getPositions().size()); - Assert.assertTrue(1 == bitmap.toArray().length); - Assert.assertTrue(5000 == bitmap.getPositions().get(0).intValue()); - bitmap.set(5001); - bitmap.set(5005); - bitmap.set(5100); - bitmap.set(5500); - bitmap.clear(); - bitmap.set(5); - bitmap.set(7); - bitmap.set(1000); - bitmap.set(1001); - Assert.assertTrue(4 == bitmap.cardinality()); - List positions = bitmap.getPositions(); - Assert.assertTrue(4 == positions.size()); - Assert.assertTrue(5 == positions.get(0).intValue()); - Assert.assertTrue(7 == positions.get(1).intValue()); - Assert.assertTrue(1000 == positions.get(2).intValue()); - Assert.assertTrue(1001 == positions.get(3).intValue()); - } - - /** - * Test ewah compressed bitmap. - */ - @Test - public void testEWAHCompressedBitmap() { - System.out.println("testing EWAH"); - long zero = 0; - long specialval = 1l | (1l << 4) | (1l << 63); - long notzero = ~zero; - EWAHCompressedBitmap myarray1 = new EWAHCompressedBitmap(); - myarray1.add(zero); - myarray1.add(zero); - myarray1.add(zero); - myarray1.add(specialval); - myarray1.add(specialval); - myarray1.add(notzero); - myarray1.add(zero); - Assert.assertEquals(myarray1.getPositions().size(), 6 + 64); - EWAHCompressedBitmap myarray2 = new EWAHCompressedBitmap(); - myarray2.add(zero); - myarray2.add(specialval); - myarray2.add(specialval); - myarray2.add(notzero); - myarray2.add(zero); - myarray2.add(zero); - myarray2.add(zero); - Assert.assertEquals(myarray2.getPositions().size(), 6 + 64); - List data1 = myarray1.getPositions(); - List data2 = myarray2.getPositions(); - Vector logicalor = new Vector(); - { - HashSet tmp = new HashSet(); - tmp.addAll(data1); - tmp.addAll(data2); - logicalor.addAll(tmp); - } - Collections.sort(logicalor); - Vector logicaland = new Vector(); - logicaland.addAll(data1); - logicaland.retainAll(data2); - Collections.sort(logicaland); - EWAHCompressedBitmap arrayand = myarray1.and(myarray2); - Assert.assertTrue(arrayand.getPositions().equals(logicaland)); - EWAHCompressedBitmap arrayor = myarray1.or(myarray2); - Assert.assertTrue(arrayor.getPositions().equals(logicalor)); - EWAHCompressedBitmap arrayandbis = myarray2.and(myarray1); - Assert.assertTrue(arrayandbis.getPositions().equals(logicaland)); - EWAHCompressedBitmap arrayorbis = myarray2.or(myarray1); - Assert.assertTrue(arrayorbis.getPositions().equals(logicalor)); - EWAHCompressedBitmap x = new EWAHCompressedBitmap(); - for (Integer i : myarray1.getPositions()) { - x.set(i.intValue()); - } - Assert.assertTrue(x.getPositions().equals(myarray1.getPositions())); - x = new EWAHCompressedBitmap(); - for (Integer i : myarray2.getPositions()) { - x.set(i.intValue()); - } - Assert.assertTrue(x.getPositions().equals(myarray2.getPositions())); - x = new EWAHCompressedBitmap(); - for (Iterator k = myarray1.iterator(); k.hasNext();) { - x.set(extracted(k).intValue()); - } - Assert.assertTrue(x.getPositions().equals(myarray1.getPositions())); - x = new EWAHCompressedBitmap(); - for (Iterator k = myarray2.iterator(); k.hasNext();) { - x.set(extracted(k).intValue()); - } - Assert.assertTrue(x.getPositions().equals(myarray2.getPositions())); - } - - /** - * Test externalization. - * - * @throws IOException Signals that an I/O exception has occurred. - */ - @Test - public void testExternalization() throws IOException { - System.out.println("testing EWAH externalization"); - EWAHCompressedBitmap ewcb = new EWAHCompressedBitmap(); - int[] val = { 5, 4400, 44600, 55400, 1000000 }; - for (int k = 0; k < val.length; ++k) { - ewcb.set(val[k]); - } - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - ObjectOutputStream oo = new ObjectOutputStream(bos); - ewcb.writeExternal(oo); - oo.close(); - ewcb = null; - ewcb = new EWAHCompressedBitmap(); - ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); - ewcb.readExternal(new ObjectInputStream(bis)); - List result = ewcb.getPositions(); - Assert.assertTrue(val.length == result.size()); - for (int k = 0; k < val.length; ++k) { - Assert.assertTrue(result.get(k).intValue() == val[k]); - } - } - - @Test - public void testExtremeRange() { - System.out.println("testing EWAH at its extreme range"); - int N = 1024; - EWAHCompressedBitmap myarray1 = new EWAHCompressedBitmap(); - for (int i = 0; i < N; ++i) { - myarray1.set(Integer.MAX_VALUE - 64 - N + i); - Assert.assertTrue(myarray1.cardinality() == i+1); - int[] val = myarray1.toArray(); - Assert.assertTrue(val[0] == Integer.MAX_VALUE - 64 - N); - } - } - - /** - * Test the intersects method - */ - @Test - public void testIntersectsMethod(){ - System.out.println("testing Intersets Bug"); - EWAHCompressedBitmap bitmap = new EWAHCompressedBitmap(); - bitmap.set(1); - EWAHCompressedBitmap bitmap2 = new EWAHCompressedBitmap(); - bitmap2.set(1); - bitmap2.set(11); - bitmap2.set(111); - bitmap2.set(1111111); - bitmap2.set(11111111); - Assert.assertTrue(bitmap.intersects(bitmap2)); - Assert.assertTrue(bitmap2.intersects(bitmap)); - - EWAHCompressedBitmap bitmap3 = new EWAHCompressedBitmap(); - bitmap3.set(101); - EWAHCompressedBitmap bitmap4 = new EWAHCompressedBitmap(); - for (int i = 0; i < 100; i++) { - bitmap4.set(i); - } - Assert.assertFalse(bitmap3.intersects(bitmap4)); - Assert.assertFalse(bitmap4.intersects(bitmap3)); - - EWAHCompressedBitmap bitmap5 = new EWAHCompressedBitmap(); - bitmap5.set(0); - bitmap5.set(10); - bitmap5.set(20); - EWAHCompressedBitmap bitmap6 = new EWAHCompressedBitmap(); - bitmap6.set(1); - bitmap6.set(11); - bitmap6.set(21); - bitmap6.set(1111111); - bitmap6.set(11111111); - Assert.assertFalse(bitmap5.intersects(bitmap6)); - Assert.assertFalse(bitmap6.intersects(bitmap5)); - - bitmap5.set(21); - Assert.assertTrue(bitmap5.intersects(bitmap6)); - Assert.assertTrue(bitmap6.intersects(bitmap5)); - - EWAHCompressedBitmap bitmap7 = new EWAHCompressedBitmap(); - bitmap7.set(1); - bitmap7.set(10); - bitmap7.set(20); - bitmap7.set(1111111); - bitmap7.set(11111111); - EWAHCompressedBitmap bitmap8 = new EWAHCompressedBitmap(); - for (int i = 0; i < 1000; i++) { - if (i != 1 && i!=10 && i!=20){ - bitmap8.set(i); - } - } - Assert.assertFalse(bitmap7.intersects(bitmap8)); - Assert.assertFalse(bitmap8.intersects(bitmap7)); - } - - /** - * as per renaud.delbru, Feb 12, 2009 this might throw an error out of bound - * exception. - */ - @Test - public void testLargeEWAHCompressedBitmap() { - System.out.println("testing EWAH over a large array"); - EWAHCompressedBitmap myarray1 = new EWAHCompressedBitmap(); - int N = 11000000; - for (int i = 0; i < N; ++i) { - myarray1.set(i); - } - Assert.assertTrue(myarray1.sizeInBits() == N); - } - - - /** - * Test massive and. - */ - @Test - public void testMassiveAnd() { - System.out.println("testing massive logical and"); - EWAHCompressedBitmap[] ewah = new EWAHCompressedBitmap[1024]; - for (int k = 0; k < ewah.length; ++k) - ewah[k] = new EWAHCompressedBitmap(); - for (int k = 0; k < 30000; ++k) { - ewah[(k + 2 * k * k) % ewah.length].set(k); - } - EWAHCompressedBitmap answer = ewah[0]; - for (int k = 1; k < ewah.length; ++k) - answer = answer.and(ewah[k]); - // result should be empty - if (answer.getPositions().size() != 0) - System.out.println(answer.toDebugString()); - Assert.assertTrue(answer.getPositions().size() == 0); - Assert.assertTrue(EWAHCompressedBitmap.and(ewah).getPositions().size() == 0); - } - - /** - * Test massive and not. - */ - @Test - public void testMassiveAndNot() { - System.out.println("testing massive and not"); - final int N = 1024; - EWAHCompressedBitmap[] ewah = new EWAHCompressedBitmap[N]; - for (int k = 0; k < ewah.length; ++k) - ewah[k] = new EWAHCompressedBitmap(); - for (int k = 0; k < 30000; ++k) { - ewah[(k + 2 * k * k) % ewah.length].set(k); - } - EWAHCompressedBitmap answer = ewah[0]; - EWAHCompressedBitmap answer2 = ewah[0]; - for (int k = 1; k < ewah.length; ++k) { - answer = answer.andNot(ewah[k]); - EWAHCompressedBitmap copy = null; - try { - copy = (EWAHCompressedBitmap) ewah[k].clone(); - copy.not(); - answer2.and(copy); - assertEqualsPositions(answer, answer2); - } catch (CloneNotSupportedException e) { - e.printStackTrace(); - } - } - } - - /** - * Test massive or. - */ - @Test - public void testMassiveOr() { - System.out.println("testing massive logical or (can take a couple of minutes)"); - final int N = 128; - for (int howmany = 512; howmany <= 10000; howmany *= 2) { - EWAHCompressedBitmap[] ewah = new EWAHCompressedBitmap[N]; - BitSet[] bset = new BitSet[N]; - for (int k = 0; k < ewah.length; ++k) - ewah[k] = new EWAHCompressedBitmap(); - for (int k = 0; k < bset.length; ++k) - bset[k] = new BitSet(); - for (int k = 0; k < N; ++k) - assertEqualsPositions(bset[k], ewah[k]); - for (int k = 0; k < howmany; ++k) { - ewah[(k + 2 * k * k) % ewah.length].set(k); - bset[(k + 2 * k * k) % ewah.length].set(k); - } - for (int k = 0; k < N; ++k) - assertEqualsPositions(bset[k], ewah[k]); - EWAHCompressedBitmap answer = ewah[0]; - BitSet bitsetanswer = bset[0]; - for (int k = 1; k < ewah.length; ++k) { - EWAHCompressedBitmap tmp = answer.or(ewah[k]); - bitsetanswer.or(bset[k]); - answer = tmp; - assertEqualsPositions(bitsetanswer, answer); - } - assertEqualsPositions(bitsetanswer, answer); - assertEqualsPositions(bitsetanswer, EWAHCompressedBitmap.or(ewah)); - int k = 0; - for (int j : answer) { - if (k != j) - System.out.println(answer.toDebugString()); - Assert.assertEquals(k, j); - k += 1; - } - } - } - - @Test - public void testsetSizeInBits() { - System.out.println("testing setSizeInBits"); - for(int k = 0; k < 4096; ++k) { - EWAHCompressedBitmap ewah = new EWAHCompressedBitmap(); - ewah.setSizeInBits(k); - Assert.assertEquals(ewah.sizeinbits,k); - Assert.assertEquals(ewah.cardinality(),0); - EWAHCompressedBitmap ewah2 = new EWAHCompressedBitmap(); - ewah2.setSizeInBits(k, false); - Assert.assertEquals(ewah2.sizeinbits,k); - Assert.assertEquals(ewah2.cardinality(),0); - EWAHCompressedBitmap ewah3 = new EWAHCompressedBitmap(); - for(int i = 0; i < k ; ++i) { - ewah3.set(i); - } - Assert.assertEquals(ewah3.sizeinbits,k); - Assert.assertEquals(ewah3.cardinality(),k); - EWAHCompressedBitmap ewah4 = new EWAHCompressedBitmap(); - ewah4.setSizeInBits(k, true); - Assert.assertEquals(ewah4.sizeinbits,k); - Assert.assertEquals(ewah4.cardinality(),k); - } - } - - - /** - * Test massive xor. - */ - @Test - public void testMassiveXOR() { - System.out.println("testing massive xor (can take a couple of minutes)"); - final int N = 16; - EWAHCompressedBitmap[] ewah = new EWAHCompressedBitmap[N]; - BitSet[] bset = new BitSet[N]; - for (int k = 0; k < ewah.length; ++k) - ewah[k] = new EWAHCompressedBitmap(); - for (int k = 0; k < bset.length; ++k) - bset[k] = new BitSet(); - for (int k = 0; k < 30000; ++k) { - ewah[(k + 2 * k * k) % ewah.length].set(k); - bset[(k + 2 * k * k) % ewah.length].set(k); - } - EWAHCompressedBitmap answer = ewah[0]; - BitSet bitsetanswer = bset[0]; - for (int k = 1; k < ewah.length; ++k) { - answer = answer.xor(ewah[k]); - bitsetanswer.xor(bset[k]); - assertEqualsPositions(bitsetanswer, answer); - } - int k = 0; - for (int j : answer) { - if (k != j) - System.out.println(answer.toDebugString()); - Assert.assertEquals(k, j); - k += 1; - } - } - - - - @Test - public void testMultiAnd() - { - System.out.println("testing MultiAnd"); - // test bitmap3 has a literal word while bitmap1/2 have a run of 1 - EWAHCompressedBitmap bitmap1 = new EWAHCompressedBitmap(); - bitmap1.addStreamOfEmptyWords(true, 1000); - EWAHCompressedBitmap bitmap2 = new EWAHCompressedBitmap(); - bitmap2.addStreamOfEmptyWords(true, 2000); - EWAHCompressedBitmap bitmap3 = new EWAHCompressedBitmap(); - bitmap3.set(500); - bitmap3.set(502); - bitmap3.set(504); - - assertAndEquals(bitmap1,bitmap2,bitmap3); - - //equal - bitmap1 = new EWAHCompressedBitmap(); - bitmap1.set(35); - bitmap2 = new EWAHCompressedBitmap(); - bitmap2.set(35); - bitmap3 = new EWAHCompressedBitmap(); - bitmap3.set(35); - - assertAndEquals(bitmap1,bitmap2,bitmap3); - - // same number of words for each - bitmap3.set(63); - assertAndEquals(bitmap1,bitmap2,bitmap3); - - // one word bigger - bitmap3.set(64); - assertAndEquals(bitmap1,bitmap2,bitmap3); - - // two words bigger - bitmap3.set(130); - assertAndEquals(bitmap1,bitmap2,bitmap3); - - // test that result can still be appended to - EWAHCompressedBitmap resultBitmap = EWAHCompressedBitmap.and(bitmap1,bitmap2,bitmap3); - resultBitmap.set(131); - - bitmap1.set(131); - assertEquals(bitmap1,resultBitmap); - - final int N = 128; - for (int howmany = 512; howmany <= 10000; howmany *= 2) { - EWAHCompressedBitmap[] ewah = new EWAHCompressedBitmap[N]; - for (int k = 0; k < ewah.length; ++k) - ewah[k] = new EWAHCompressedBitmap(); - for (int k = 0; k < howmany; ++k) { - ewah[(k + 2 * k * k) % ewah.length].set(k); - } - for(int k = 1; k<=ewah.length;++k) { - EWAHCompressedBitmap[] shortewah = new EWAHCompressedBitmap[k]; - for(int i = 0; i= 0; rl -= 1024) { - rlw.setNumberOfLiteralWords(rl); - Assert.assertEquals(rl, rlw.getNumberOfLiteralWords()); - Assert.assertEquals(false, rlw.getRunningBit()); - Assert.assertEquals(0, rlw.getRunningLength()); - rlw.setNumberOfLiteralWords(0); - Assert.assertEquals(0, rlw.getNumberOfLiteralWords()); - Assert.assertEquals(false, rlw.getRunningBit()); - Assert.assertEquals(0, rlw.getRunningLength()); - } - for (long rl = 0; rl <= RunningLengthWord.largestrunninglengthcount; rl += 1024) { - rlw.setRunningLength(rl); - Assert.assertEquals(0, rlw.getNumberOfLiteralWords()); - Assert.assertEquals(false, rlw.getRunningBit()); - Assert.assertEquals(rl, rlw.getRunningLength()); - rlw.setRunningLength(0); - Assert.assertEquals(0, rlw.getNumberOfLiteralWords()); - Assert.assertEquals(false, rlw.getRunningBit()); - Assert.assertEquals(0, rlw.getRunningLength()); - } - rlw.setRunningBit(true); - for (long rl = 0; rl <= RunningLengthWord.largestrunninglengthcount; rl += 1024) { - rlw.setRunningLength(rl); - Assert.assertEquals(0, rlw.getNumberOfLiteralWords()); - Assert.assertEquals(true, rlw.getRunningBit()); - Assert.assertEquals(rl, rlw.getRunningLength()); - rlw.setRunningLength(0); - Assert.assertEquals(0, rlw.getNumberOfLiteralWords()); - Assert.assertEquals(true, rlw.getRunningBit()); - Assert.assertEquals(0, rlw.getRunningLength()); - } - for (long rl = 0; rl <= RunningLengthWord.largestliteralcount; rl += 128) { - rlw.setNumberOfLiteralWords(rl); - Assert.assertEquals(rl, rlw.getNumberOfLiteralWords()); - Assert.assertEquals(true, rlw.getRunningBit()); - Assert.assertEquals(0, rlw.getRunningLength()); - rlw.setNumberOfLiteralWords(0); - Assert.assertEquals(0, rlw.getNumberOfLiteralWords()); - Assert.assertEquals(true, rlw.getRunningBit()); - Assert.assertEquals(0, rlw.getRunningLength()); - } - } - - /** - * Test sets and gets. - */ - @Test - public void testSetGet() { - System.out.println("testing EWAH set/get"); - EWAHCompressedBitmap ewcb = new EWAHCompressedBitmap(); - int[] val = { 5, 4400, 44600, 55400, 1000000 }; - for (int k = 0; k < val.length; ++k) { - ewcb.set(val[k]); - } - List result = ewcb.getPositions(); - Assert.assertTrue(val.length == result.size()); - for (int k = 0; k < val.length; ++k) { - Assert.assertEquals(result.get(k).intValue() , val[k]); - } - } - - @Test - public void testHashCode() { - System.out.println("testing hashCode"); - EWAHCompressedBitmap ewcb = EWAHCompressedBitmap.bitmapOf(50, 70).and(EWAHCompressedBitmap.bitmapOf(50, 1000)); - Assert.assertEquals(EWAHCompressedBitmap.bitmapOf(50), ewcb); - Assert.assertEquals(EWAHCompressedBitmap.bitmapOf(50).hashCode(), ewcb.hashCode()); - } - - @Test - public void testSetSizeInBits() - { - System.out.println("testing SetSizeInBits"); - testSetSizeInBits(130,131); - testSetSizeInBits(63,64); - testSetSizeInBits(64,65); - testSetSizeInBits(64,128); - testSetSizeInBits(35,131); - testSetSizeInBits(130,400); - testSetSizeInBits(130,191); - testSetSizeInBits(130,192); - EWAHCompressedBitmap bitmap = new EWAHCompressedBitmap(); - bitmap.set(31); - bitmap.setSizeInBits(130,false); - bitmap.set(131); - BitSet jdkBitmap = new BitSet(); - jdkBitmap.set(31); - jdkBitmap.set(131); - assertEquals(jdkBitmap,bitmap); - } - /** - * Test with parameters. - * - * @throws IOException Signals that an I/O exception has occurred. - */ - @Test - public void testWithParameters() throws IOException { - System.out - .println("These tests can run for several minutes. Please be patient."); - for (int k = 2; k < 1 << 24; k *= 8) - shouldSetBits(k); - PolizziTest(64); - PolizziTest(128); - PolizziTest(256); - PolizziTest(2048); - System.out.println("Your code is probably ok."); - } - - /** - * Pseudo-non-deterministic test inspired by S.J.vanSchaik. - * (Yes, non-deterministic tests are bad, but the test is actually deterministic.) - */ - @Test - public void vanSchaikTest() { - System.out.println("testing vanSchaikTest (this takes some time)"); - final int totalNumBits = 32768; - final double odds = 0.9; - Random rand = new Random(323232323); - for (int t = 0; t < 100; t++) { - int numBitsSet = 0; - EWAHCompressedBitmap cBitMap = new EWAHCompressedBitmap(); - for (int i = 0; i < totalNumBits; i++) { - if (rand.nextDouble() < odds) { - cBitMap.set(i); - numBitsSet++; + + @Test + public void testGet() { + for (int gap = 29; gap < 10000; gap *= 10) { + EWAHCompressedBitmap x = new EWAHCompressedBitmap(); + for (int k = 0; k < 100; ++k) + x.set(k * gap); + for (int k = 0; k < 100 * gap; ++k) + if (x.get(k)) { + if (k % gap != 0) + throw new RuntimeException( + "spotted an extra set bit at " + + k + " gap = " + + gap); + } else if (k % gap == 0) + throw new RuntimeException( + "missed a set bit " + k + + " gap = " + gap); + } } - } - Assert.assertEquals(cBitMap.cardinality(),numBitsSet); - } - - } - - /** - * Function used in a test inspired by Federico Fissore. - * - * @param size the number of set bits - * @param seed the random seed - * @return the pseudo-random array int[] - */ - public static int[] createSortedIntArrayOfBitsToSet(int size, int seed) { - Random random = new Random(seed); - // build raw int array - int[] bits = new int[size]; - for (int i = 0; i < bits.length; i++) { - bits[i] = random.nextInt(TEST_BS_SIZE); - } - // might generate duplicates - Arrays.sort(bits); - // first count how many distinct values - int counter = 0; - int oldx = -1; - for (int x : bits) { - if (x != oldx) - ++counter; - oldx = x; - } - // then construct new array - int[] answer = new int[counter]; - counter = 0; - oldx = -1; - for (int x : bits) { - if (x != oldx) { - answer[counter] = x; - ++counter; - } - oldx = x; - } - return answer; - } - /** - * Test inspired by Bilal Tayara - */ - @Test - public void TayaraTest() { - System.out.println("Tayara test"); - for(int offset = 64; offset<(1<<30);offset*=2){ - EWAHCompressedBitmap a = new EWAHCompressedBitmap(); - EWAHCompressedBitmap b = new EWAHCompressedBitmap(); - for(int k = 0; k< 64; ++k) { - a.set(offset+k); - b.set(offset+k); - } - if(!a.and(b).equals(a)) throw new RuntimeException("bug"); - if(!a.or(b).equals(a)) throw new RuntimeException("bug"); - } - } - - @Test + + @SuppressWarnings({ "deprecation", "boxing" }) + @Test + public void OKaserBugReportJuly2013() { + System.out.println("testing OKaserBugReportJuly2013"); + int[][] data = { {}, { 5, 6, 7, 8, 9 }, { 1 }, { 2 }, { 2, 5, 7 }, + { 1 }, { 2 }, { 1, 6, 9 }, { 1, 3, 4, 6, 8, 9 }, + { 1, 3, 4, 6, 8, 9 }, { 1, 3, 6, 8, 9 }, { 2, 5, 7 }, + { 2, 5, 7 }, { 1, 3, 9 }, { 3, 8, 9 } }; + + EWAHCompressedBitmap[] toBeOred = new EWAHCompressedBitmap[data.length]; + Set bruteForceAnswer = new HashSet(); + for (int i = 0; i < toBeOred.length; ++i) { + toBeOred[i] = new EWAHCompressedBitmap(); + for (int j : data[i]) { + toBeOred[i].set(j); + bruteForceAnswer.add(j); + } + toBeOred[i].setSizeInBits(1000,false); + } + long rightcard = bruteForceAnswer.size(); + EWAHCompressedBitmap e1 = FastAggregation.or(toBeOred); + Assert.assertEquals(rightcard, e1.cardinality()); + EWAHCompressedBitmap e2 = FastAggregation.bufferedor(65536, toBeOred); + Assert.assertEquals(rightcard, e2.cardinality()); + EWAHCompressedBitmap foo = new EWAHCompressedBitmap(); + FastAggregation.legacy_orWithContainer(foo, toBeOred); + Assert.assertEquals(rightcard, foo.cardinality()); + } + + + @Test + public void testSizeInBitsWithAnd() { + System.out.println("testing SizeInBitsWithAnd"); + EWAHCompressedBitmap a = new EWAHCompressedBitmap(); + EWAHCompressedBitmap b = new EWAHCompressedBitmap(); + + a.set(1); + a.set(2); + a.set(3); + + b.set(3); + b.set(4); + b.set(5); + + a.setSizeInBits(10,false); + b.setSizeInBits(10,false); + + EWAHCompressedBitmap and = a.and(b); + Assert.assertEquals(10, and.sizeInBits()); + EWAHCompressedBitmap and2 = EWAHCompressedBitmap.and(a,b); + Assert.assertEquals(10, and2.sizeInBits()); + } + @Test + public void testSizeInBitsWithAndNot() { + System.out.println("testing SizeInBitsWithAndNot"); + EWAHCompressedBitmap a = new EWAHCompressedBitmap(); + EWAHCompressedBitmap b = new EWAHCompressedBitmap(); + + a.set(1); + a.set(2); + a.set(3); + + b.set(3); + b.set(4); + b.set(5); + + a.setSizeInBits(10,false); + b.setSizeInBits(10,false); + + EWAHCompressedBitmap and = a.andNot(b); + Assert.assertEquals(10, and.sizeInBits()); + } + + @Test + public void testSizeInBitsWithOr() { + System.out.println("testing SizeInBitsWithOr"); + EWAHCompressedBitmap a = new EWAHCompressedBitmap(); + EWAHCompressedBitmap b = new EWAHCompressedBitmap(); + + a.set(1); + a.set(2); + a.set(3); + + b.set(3); + b.set(4); + b.set(5); + + a.setSizeInBits(10,false); + b.setSizeInBits(10,false); + + EWAHCompressedBitmap or = a.or(b); + Assert.assertEquals(10, or.sizeInBits()); + EWAHCompressedBitmap or2 = EWAHCompressedBitmap.or(a,b); + Assert.assertEquals(10, or2.sizeInBits()); + } + + + @Test + public void testSizeInBitsWithXor() { + System.out.println("testing SizeInBitsWithXor"); + EWAHCompressedBitmap a = new EWAHCompressedBitmap(); + EWAHCompressedBitmap b = new EWAHCompressedBitmap(); + + a.set(1); + a.set(2); + a.set(3); + + b.set(3); + b.set(4); + b.set(5); + + a.setSizeInBits(10,false); + b.setSizeInBits(10,false); + + EWAHCompressedBitmap xor = a.xor(b); + Assert.assertEquals(10, xor.sizeInBits()); + EWAHCompressedBitmap xor2 = EWAHCompressedBitmap.xor(a,b); + Assert.assertEquals(10, xor2.sizeInBits()); + } + + + @Test + public void testDebugSetSizeInBitsTest() { + System.out.println("testing DebugSetSizeInBits"); + EWAHCompressedBitmap b = new EWAHCompressedBitmap(); + + b.set(4); + + b.setSizeInBits(6, true); + + List positions = b.getPositions(); + + Assert.assertEquals(2, positions.size()); + Assert.assertEquals(Integer.valueOf(4), positions.get(0)); + Assert.assertEquals(Integer.valueOf(5), positions.get(1)); + + Iterator iterator = b.iterator(); + Assert.assertTrue(iterator.hasNext()); + Assert.assertEquals(Integer.valueOf(4), iterator.next()); + Assert.assertTrue(iterator.hasNext()); + Assert.assertEquals(Integer.valueOf(5), iterator.next()); + Assert.assertFalse(iterator.hasNext()); + + IntIterator intIterator = b.intIterator(); + Assert.assertTrue(intIterator.hasNext()); + Assert.assertEquals(4, intIterator.next()); + Assert.assertTrue(intIterator.hasNext()); + Assert.assertEquals(5, intIterator.next()); + Assert.assertFalse(intIterator.hasNext()); + + } + + /** + * Created: 2/4/11 6:03 PM By: Arnon Moscona. + */ + @Test + public void EwahIteratorProblem() { + System.out.println("testing ArnonMoscona"); + EWAHCompressedBitmap bitmap = new EWAHCompressedBitmap(); + for (int i = 9434560; i <= 9435159; i++) { + bitmap.set(i); + } + IntIterator iterator = bitmap.intIterator(); + List v = bitmap.getPositions(); + int[] array = bitmap.toArray(); + for (int k = 0; k < v.size(); ++k) { + Assert.assertTrue(array[k] == v.get(k).intValue()); + Assert.assertTrue(iterator.hasNext()); + final int ival = iterator.next(); + final int vval = v.get(k).intValue(); + Assert.assertTrue(ival == vval); + } + Assert.assertTrue(!iterator.hasNext()); + // + for (int k = 2; k <= 1024; k *= 2) { + int[] bitsToSet = createSortedIntArrayOfBitsToSet(k, 434455 + 5 * k); + EWAHCompressedBitmap ewah = new EWAHCompressedBitmap(); + for (int i : bitsToSet) { + ewah.set(i); + } + equal(ewah.iterator(), bitsToSet); + } + } + + /** + * Test submitted by Gregory Ssi-Yan-Kai + */ + @Test + public void SsiYanKaiTest() { + System.out.println("testing SsiYanKaiTest"); + EWAHCompressedBitmap a = EWAHCompressedBitmap.bitmapOf(39935, 39936, + 39937, 39938, 39939, 39940, 39941, 39942, 39943, 39944, 39945, + 39946, 39947, 39948, 39949, 39950, 39951, 39952, 39953, 39954, + 39955, 39956, 39957, 39958, 39959, 39960, 39961, 39962, 39963, + 39964, 39965, 39966, 39967, 39968, 39969, 39970, 39971, 39972, + 39973, 39974, 39975, 39976, 39977, 39978, 39979, 39980, 39981, + 39982, 39983, 39984, 39985, 39986, 39987, 39988, 39989, 39990, + 39991, 39992, 39993, 39994, 39995, 39996, 39997, 39998, 39999, + 40000, 40001, 40002, 40003, 40004, 40005, 40006, 40007, 40008, + 40009, 40010, 40011, 40012, 40013, 40014, 40015, 40016, 40017, + 40018, 40019, 40020, 40021, 40022, 40023, 40024, 40025, 40026, + 40027, 40028, 40029, 40030, 40031, 40032, 40033, 40034, 40035, + 40036, 40037, 40038, 40039, 40040, 40041, 40042, 40043, 40044, + 40045, 40046, 40047, 40048, 40049, 40050, 40051, 40052, 40053, + 40054, 40055, 40056, 40057, 40058, 40059, 40060, 40061, 40062, + 40063, 40064, 40065, 40066, 40067, 40068, 40069, 40070, 40071, + 40072, 40073, 40074, 40075, 40076, 40077, 40078, 40079, 40080, + 40081, 40082, 40083, 40084, 40085, 40086, 40087, 40088, 40089, + 40090, 40091, 40092, 40093, 40094, 40095, 40096, 40097, 40098, + 40099, 40100); + EWAHCompressedBitmap b = EWAHCompressedBitmap.bitmapOf(39935, 39936, + 39937, 39938, 39939, 39940, 39941, 39942, 39943, 39944, 39945, + 39946, 39947, 39948, 39949, 39950, 39951, 39952, 39953, 39954, + 39955, 39956, 39957, 39958, 39959, 39960, 39961, 39962, 39963, + 39964, 39965, 39966, 39967, 39968, 39969, 39970, 39971, 39972, + 39973, 39974, 39975, 39976, 39977, 39978, 39979, 39980, 39981, + 39982, 39983, 39984, 39985, 39986, 39987, 39988, 39989, 39990, + 39991, 39992, 39993, 39994, 39995, 39996, 39997, 39998, 39999, + 270000); + LinkedHashSet aPositions = new LinkedHashSet( + a.getPositions()); + int intersection = 0; + EWAHCompressedBitmap inter = new EWAHCompressedBitmap(); + LinkedHashSet bPositions = new LinkedHashSet( + b.getPositions()); + for (Integer integer : bPositions) { + if (aPositions.contains(integer)) { + inter.set(integer.intValue()); + ++intersection; + } + } + EWAHCompressedBitmap and2 = a.and(b); + if (!and2.equals(inter)) + throw new RuntimeException("intersections don't match"); + if (intersection != and2.cardinality()) + throw new RuntimeException("cardinalities don't match"); + } + + /** + * Test inspired by William Habermaas. + */ + @Test + public void habermaasTest() { + System.out.println("testing habermaasTest"); + BitSet bitsetaa = new BitSet(); + EWAHCompressedBitmap aa = new EWAHCompressedBitmap(); + int[] val = { 55400, 1000000, 1000128 }; + for (int k = 0; k < val.length; ++k) { + aa.set(val[k]); + bitsetaa.set(val[k]); + } + equal(aa, bitsetaa); + BitSet bitsetab = new BitSet(); + EWAHCompressedBitmap ab = new EWAHCompressedBitmap(); + for (int i = 4096; i < (4096 + 5); i++) { + ab.set(i); + bitsetab.set(i); + } + ab.set(99000); + bitsetab.set(99000); + ab.set(1000130); + bitsetab.set(1000130); + equal(ab, bitsetab); + EWAHCompressedBitmap bb = aa.or(ab); + EWAHCompressedBitmap bbAnd = aa.and(ab); + try { + EWAHCompressedBitmap abnot = ab.clone(); + abnot.not(); + EWAHCompressedBitmap bbAnd2 = aa.andNot(abnot); + assertEquals(bbAnd2, bbAnd); + } catch (CloneNotSupportedException e) { + e.printStackTrace(); + } + BitSet bitsetbb = (BitSet) bitsetaa.clone(); + bitsetbb.or(bitsetab); + BitSet bitsetbbAnd = (BitSet) bitsetaa.clone(); + bitsetbbAnd.and(bitsetab); + equal(bbAnd, bitsetbbAnd); + equal(bb, bitsetbb); + } + + @Test + public void testAndResultAppend() { + System.out.println("testing AndResultAppend"); + EWAHCompressedBitmap bitmap1 = new EWAHCompressedBitmap(); + bitmap1.set(35); + EWAHCompressedBitmap bitmap2 = new EWAHCompressedBitmap(); + bitmap2.set(35); + bitmap2.set(130); + + EWAHCompressedBitmap resultBitmap = bitmap1.and(bitmap2); + resultBitmap.set(131); + + bitmap1.set(131); + assertEquals(bitmap1, resultBitmap); + } + + /** + * Test cardinality. + */ + @Test + public void testCardinality() { + System.out.println("testing EWAH cardinality"); + EWAHCompressedBitmap bitmap = new EWAHCompressedBitmap(); + bitmap.set(Integer.MAX_VALUE - 64); + // System.out.format("Total Items %d\n", bitmap.cardinality()); + Assert.assertTrue(bitmap.cardinality() == 1); + } + + /** + * Test clear function + */ + @Test + public void testClear() { + System.out.println("testing Clear"); + EWAHCompressedBitmap bitmap = new EWAHCompressedBitmap(); + bitmap.set(5); + bitmap.clear(); + bitmap.set(7); + Assert.assertTrue(1 == bitmap.cardinality()); + Assert.assertTrue(1 == bitmap.getPositions().size()); + Assert.assertTrue(1 == bitmap.toArray().length); + Assert.assertTrue(7 == bitmap.getPositions().get(0).intValue()); + Assert.assertTrue(7 == bitmap.toArray()[0]); + bitmap.clear(); + bitmap.set(5000); + Assert.assertTrue(1 == bitmap.cardinality()); + Assert.assertTrue(1 == bitmap.getPositions().size()); + Assert.assertTrue(1 == bitmap.toArray().length); + Assert.assertTrue(5000 == bitmap.getPositions().get(0).intValue()); + bitmap.set(5001); + bitmap.set(5005); + bitmap.set(5100); + bitmap.set(5500); + bitmap.clear(); + bitmap.set(5); + bitmap.set(7); + bitmap.set(1000); + bitmap.set(1001); + Assert.assertTrue(4 == bitmap.cardinality()); + List positions = bitmap.getPositions(); + Assert.assertTrue(4 == positions.size()); + Assert.assertTrue(5 == positions.get(0).intValue()); + Assert.assertTrue(7 == positions.get(1).intValue()); + Assert.assertTrue(1000 == positions.get(2).intValue()); + Assert.assertTrue(1001 == positions.get(3).intValue()); + } + + /** + * Test ewah compressed bitmap. + */ + @Test + public void testEWAHCompressedBitmap() { + System.out.println("testing EWAH"); + long zero = 0; + long specialval = 1l | (1l << 4) | (1l << 63); + long notzero = ~zero; + EWAHCompressedBitmap myarray1 = new EWAHCompressedBitmap(); + myarray1.add(zero); + myarray1.add(zero); + myarray1.add(zero); + myarray1.add(specialval); + myarray1.add(specialval); + myarray1.add(notzero); + myarray1.add(zero); + Assert.assertEquals(myarray1.getPositions().size(), 6 + 64); + EWAHCompressedBitmap myarray2 = new EWAHCompressedBitmap(); + myarray2.add(zero); + myarray2.add(specialval); + myarray2.add(specialval); + myarray2.add(notzero); + myarray2.add(zero); + myarray2.add(zero); + myarray2.add(zero); + Assert.assertEquals(myarray2.getPositions().size(), 6 + 64); + List data1 = myarray1.getPositions(); + List data2 = myarray2.getPositions(); + Vector logicalor = new Vector(); + { + HashSet tmp = new HashSet(); + tmp.addAll(data1); + tmp.addAll(data2); + logicalor.addAll(tmp); + } + Collections.sort(logicalor); + Vector logicaland = new Vector(); + logicaland.addAll(data1); + logicaland.retainAll(data2); + Collections.sort(logicaland); + EWAHCompressedBitmap arrayand = myarray1.and(myarray2); + Assert.assertTrue(arrayand.getPositions().equals(logicaland)); + EWAHCompressedBitmap arrayor = myarray1.or(myarray2); + Assert.assertTrue(arrayor.getPositions().equals(logicalor)); + EWAHCompressedBitmap arrayandbis = myarray2.and(myarray1); + Assert.assertTrue(arrayandbis.getPositions().equals(logicaland)); + EWAHCompressedBitmap arrayorbis = myarray2.or(myarray1); + Assert.assertTrue(arrayorbis.getPositions().equals(logicalor)); + EWAHCompressedBitmap x = new EWAHCompressedBitmap(); + for (Integer i : myarray1.getPositions()) { + x.set(i.intValue()); + } + Assert.assertTrue(x.getPositions().equals(myarray1.getPositions())); + x = new EWAHCompressedBitmap(); + for (Integer i : myarray2.getPositions()) { + x.set(i.intValue()); + } + Assert.assertTrue(x.getPositions().equals(myarray2.getPositions())); + x = new EWAHCompressedBitmap(); + for (Iterator k = myarray1.iterator(); k.hasNext();) { + x.set(extracted(k).intValue()); + } + Assert.assertTrue(x.getPositions().equals(myarray1.getPositions())); + x = new EWAHCompressedBitmap(); + for (Iterator k = myarray2.iterator(); k.hasNext();) { + x.set(extracted(k).intValue()); + } + Assert.assertTrue(x.getPositions().equals(myarray2.getPositions())); + } + + /** + * Test externalization. + * + * @throws IOException + * Signals that an I/O exception has occurred. + */ + @Test + public void testExternalization() throws IOException { + System.out.println("testing EWAH externalization"); + EWAHCompressedBitmap ewcb = new EWAHCompressedBitmap(); + int[] val = { 5, 4400, 44600, 55400, 1000000 }; + for (int k = 0; k < val.length; ++k) { + ewcb.set(val[k]); + } + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + ObjectOutputStream oo = new ObjectOutputStream(bos); + ewcb.writeExternal(oo); + oo.close(); + ewcb = null; + ewcb = new EWAHCompressedBitmap(); + ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); + ewcb.readExternal(new ObjectInputStream(bis)); + List result = ewcb.getPositions(); + Assert.assertTrue(val.length == result.size()); + for (int k = 0; k < val.length; ++k) { + Assert.assertTrue(result.get(k).intValue() == val[k]); + } + } + + @Test + public void testExtremeRange() { + System.out.println("testing EWAH at its extreme range"); + int N = 1024; + EWAHCompressedBitmap myarray1 = new EWAHCompressedBitmap(); + for (int i = 0; i < N; ++i) { + myarray1.set(Integer.MAX_VALUE - 64 - N + i); + Assert.assertTrue(myarray1.cardinality() == i + 1); + int[] val = myarray1.toArray(); + Assert.assertTrue(val[0] == Integer.MAX_VALUE - 64 - N); + } + } + + /** + * Test the intersects method + */ + @Test + public void testIntersectsMethod() { + System.out.println("testing Intersets Bug"); + EWAHCompressedBitmap bitmap = new EWAHCompressedBitmap(); + bitmap.set(1); + EWAHCompressedBitmap bitmap2 = new EWAHCompressedBitmap(); + bitmap2.set(1); + bitmap2.set(11); + bitmap2.set(111); + bitmap2.set(1111111); + bitmap2.set(11111111); + Assert.assertTrue(bitmap.intersects(bitmap2)); + Assert.assertTrue(bitmap2.intersects(bitmap)); + + EWAHCompressedBitmap bitmap3 = new EWAHCompressedBitmap(); + bitmap3.set(101); + EWAHCompressedBitmap bitmap4 = new EWAHCompressedBitmap(); + for (int i = 0; i < 100; i++) { + bitmap4.set(i); + } + Assert.assertFalse(bitmap3.intersects(bitmap4)); + Assert.assertFalse(bitmap4.intersects(bitmap3)); + + EWAHCompressedBitmap bitmap5 = new EWAHCompressedBitmap(); + bitmap5.set(0); + bitmap5.set(10); + bitmap5.set(20); + EWAHCompressedBitmap bitmap6 = new EWAHCompressedBitmap(); + bitmap6.set(1); + bitmap6.set(11); + bitmap6.set(21); + bitmap6.set(1111111); + bitmap6.set(11111111); + Assert.assertFalse(bitmap5.intersects(bitmap6)); + Assert.assertFalse(bitmap6.intersects(bitmap5)); + + bitmap5.set(21); + Assert.assertTrue(bitmap5.intersects(bitmap6)); + Assert.assertTrue(bitmap6.intersects(bitmap5)); + + EWAHCompressedBitmap bitmap7 = new EWAHCompressedBitmap(); + bitmap7.set(1); + bitmap7.set(10); + bitmap7.set(20); + bitmap7.set(1111111); + bitmap7.set(11111111); + EWAHCompressedBitmap bitmap8 = new EWAHCompressedBitmap(); + for (int i = 0; i < 1000; i++) { + if (i != 1 && i != 10 && i != 20) { + bitmap8.set(i); + } + } + Assert.assertFalse(bitmap7.intersects(bitmap8)); + Assert.assertFalse(bitmap8.intersects(bitmap7)); + } + + /** + * as per renaud.delbru, Feb 12, 2009 this might throw an error out of bound + * exception. + */ + @Test + public void testLargeEWAHCompressedBitmap() { + System.out.println("testing EWAH over a large array"); + EWAHCompressedBitmap myarray1 = new EWAHCompressedBitmap(); + int N = 11000000; + for (int i = 0; i < N; ++i) { + myarray1.set(i); + } + Assert.assertTrue(myarray1.sizeInBits() == N); + } + + /** + * Test massive and. + */ + @Test + public void testMassiveAnd() { + System.out.println("testing massive logical and"); + EWAHCompressedBitmap[] ewah = new EWAHCompressedBitmap[1024]; + for (int k = 0; k < ewah.length; ++k) + ewah[k] = new EWAHCompressedBitmap(); + for (int k = 0; k < 30000; ++k) { + ewah[(k + 2 * k * k) % ewah.length].set(k); + } + EWAHCompressedBitmap answer = ewah[0]; + for (int k = 1; k < ewah.length; ++k) + answer = answer.and(ewah[k]); + // result should be empty + if (answer.getPositions().size() != 0) + System.out.println(answer.toDebugString()); + Assert.assertTrue(answer.getPositions().size() == 0); + Assert.assertTrue(EWAHCompressedBitmap.and(ewah).getPositions().size() == 0); + } + + /** + * Test massive and not. + */ + @Test + public void testMassiveAndNot() { + System.out.println("testing massive and not"); + final int N = 1024; + EWAHCompressedBitmap[] ewah = new EWAHCompressedBitmap[N]; + for (int k = 0; k < ewah.length; ++k) + ewah[k] = new EWAHCompressedBitmap(); + for (int k = 0; k < 30000; ++k) { + ewah[(k + 2 * k * k) % ewah.length].set(k); + } + EWAHCompressedBitmap answer = ewah[0]; + EWAHCompressedBitmap answer2 = ewah[0]; + for (int k = 1; k < ewah.length; ++k) { + answer = answer.andNot(ewah[k]); + EWAHCompressedBitmap copy = null; + try { + copy = ewah[k].clone(); + copy.not(); + answer2.and(copy); + assertEqualsPositions(answer, answer2); + } catch (CloneNotSupportedException e) { + e.printStackTrace(); + } + } + } + + /** + * Test massive or. + */ + @Test + public void testMassiveOr() { + System.out + .println("testing massive logical or (can take a couple of minutes)"); + final int N = 128; + for (int howmany = 512; howmany <= 10000; howmany *= 2) { + EWAHCompressedBitmap[] ewah = new EWAHCompressedBitmap[N]; + BitSet[] bset = new BitSet[N]; + for (int k = 0; k < ewah.length; ++k) + ewah[k] = new EWAHCompressedBitmap(); + for (int k = 0; k < bset.length; ++k) + bset[k] = new BitSet(); + for (int k = 0; k < N; ++k) + assertEqualsPositions(bset[k], ewah[k]); + for (int k = 0; k < howmany; ++k) { + ewah[(k + 2 * k * k) % ewah.length].set(k); + bset[(k + 2 * k * k) % ewah.length].set(k); + } + for (int k = 0; k < N; ++k) + assertEqualsPositions(bset[k], ewah[k]); + EWAHCompressedBitmap answer = ewah[0]; + BitSet bitsetanswer = bset[0]; + for (int k = 1; k < ewah.length; ++k) { + EWAHCompressedBitmap tmp = answer.or(ewah[k]); + bitsetanswer.or(bset[k]); + answer = tmp; + assertEqualsPositions(bitsetanswer, answer); + } + assertEqualsPositions(bitsetanswer, answer); + assertEqualsPositions(bitsetanswer, EWAHCompressedBitmap.or(ewah)); + int k = 0; + for (int j : answer) { + if (k != j) + System.out.println(answer.toDebugString()); + Assert.assertEquals(k, j); + k += 1; + } + } + } + + @Test + public void testsetSizeInBits() { + System.out.println("testing setSizeInBits"); + for (int k = 0; k < 4096; ++k) { + EWAHCompressedBitmap ewah = new EWAHCompressedBitmap(); + ewah.setSizeInBits(k,false); + Assert.assertEquals(ewah.sizeinbits, k); + Assert.assertEquals(ewah.cardinality(), 0); + EWAHCompressedBitmap ewah2 = new EWAHCompressedBitmap(); + ewah2.setSizeInBits(k, false); + Assert.assertEquals(ewah2.sizeinbits, k); + Assert.assertEquals(ewah2.cardinality(), 0); + EWAHCompressedBitmap ewah3 = new EWAHCompressedBitmap(); + for (int i = 0; i < k; ++i) { + ewah3.set(i); + } + Assert.assertEquals(ewah3.sizeinbits, k); + Assert.assertEquals(ewah3.cardinality(), k); + EWAHCompressedBitmap ewah4 = new EWAHCompressedBitmap(); + ewah4.setSizeInBits(k, true); + Assert.assertEquals(ewah4.sizeinbits, k); + Assert.assertEquals(ewah4.cardinality(), k); + } + } + + /** + * Test massive xor. + */ + @Test + public void testMassiveXOR() { + System.out + .println("testing massive xor (can take a couple of minutes)"); + final int N = 16; + EWAHCompressedBitmap[] ewah = new EWAHCompressedBitmap[N]; + BitSet[] bset = new BitSet[N]; + for (int k = 0; k < ewah.length; ++k) + ewah[k] = new EWAHCompressedBitmap(); + for (int k = 0; k < bset.length; ++k) + bset[k] = new BitSet(); + for (int k = 0; k < 30000; ++k) { + ewah[(k + 2 * k * k) % ewah.length].set(k); + bset[(k + 2 * k * k) % ewah.length].set(k); + } + EWAHCompressedBitmap answer = ewah[0]; + BitSet bitsetanswer = bset[0]; + for (int k = 1; k < ewah.length; ++k) { + answer = answer.xor(ewah[k]); + bitsetanswer.xor(bset[k]); + assertEqualsPositions(bitsetanswer, answer); + } + int k = 0; + for (int j : answer) { + if (k != j) + System.out.println(answer.toDebugString()); + Assert.assertEquals(k, j); + k += 1; + } + } + + @Test + public void testMultiAnd() { + System.out.println("testing MultiAnd"); + // test bitmap3 has a literal word while bitmap1/2 have a run of 1 + EWAHCompressedBitmap bitmap1 = new EWAHCompressedBitmap(); + bitmap1.addStreamOfEmptyWords(true, 1000); + EWAHCompressedBitmap bitmap2 = new EWAHCompressedBitmap(); + bitmap2.addStreamOfEmptyWords(true, 2000); + EWAHCompressedBitmap bitmap3 = new EWAHCompressedBitmap(); + bitmap3.set(500); + bitmap3.set(502); + bitmap3.set(504); + + assertAndEquals(bitmap1, bitmap2, bitmap3); + + // equal + bitmap1 = new EWAHCompressedBitmap(); + bitmap1.set(35); + bitmap2 = new EWAHCompressedBitmap(); + bitmap2.set(35); + bitmap3 = new EWAHCompressedBitmap(); + bitmap3.set(35); + assertAndEquals(bitmap1, bitmap2, bitmap3); + + // same number of words for each + bitmap3.set(63); + assertAndEquals(bitmap1, bitmap2, bitmap3); + + // one word bigger + bitmap3.set(64); + assertAndEquals(bitmap1, bitmap2, bitmap3); + + // two words bigger + bitmap3.set(130); + assertAndEquals(bitmap1, bitmap2, bitmap3); + + // test that result can still be appended to + EWAHCompressedBitmap resultBitmap = EWAHCompressedBitmap.and(bitmap1, + bitmap2, bitmap3); + + resultBitmap.set(131); + + bitmap1.set(131); + assertEquals(bitmap1, resultBitmap); + + final int N = 128; + for (int howmany = 512; howmany <= 10000; howmany *= 2) { + EWAHCompressedBitmap[] ewah = new EWAHCompressedBitmap[N]; + for (int k = 0; k < ewah.length; ++k) + ewah[k] = new EWAHCompressedBitmap(); + for (int k = 0; k < howmany; ++k) { + ewah[(k + 2 * k * k) % ewah.length].set(k); + } + for (int k = 1; k <= ewah.length; ++k) { + EWAHCompressedBitmap[] shortewah = new EWAHCompressedBitmap[k]; + for (int i = 0; i < k; ++i) + shortewah[i] = ewah[i]; + assertAndEquals(shortewah); + } + } + } + + @Test + public void testMultiOr() { + System.out.println("testing MultiOr"); + // test bitmap3 has a literal word while bitmap1/2 have a run of 0 + EWAHCompressedBitmap bitmap1 = new EWAHCompressedBitmap(); + bitmap1.set(1000); + EWAHCompressedBitmap bitmap2 = new EWAHCompressedBitmap(); + bitmap2.set(2000); + EWAHCompressedBitmap bitmap3 = new EWAHCompressedBitmap(); + bitmap3.set(500); + bitmap3.set(502); + bitmap3.set(504); + + EWAHCompressedBitmap expected = bitmap1.or(bitmap2).or(bitmap3); + + assertEquals(expected, + EWAHCompressedBitmap.or(bitmap1, bitmap2, bitmap3)); + + final int N = 128; + for (int howmany = 512; howmany <= 10000; howmany *= 2) { + EWAHCompressedBitmap[] ewah = new EWAHCompressedBitmap[N]; + for (int k = 0; k < ewah.length; ++k) + ewah[k] = new EWAHCompressedBitmap(); + for (int k = 0; k < howmany; ++k) { + ewah[(k + 2 * k * k) % ewah.length].set(k); + } + for (int k = 1; k <= ewah.length; ++k) { + EWAHCompressedBitmap[] shortewah = new EWAHCompressedBitmap[k]; + for (int i = 0; i < k; ++i) + shortewah[i] = ewah[i]; + assertOrEquals(shortewah); + } + } + + } + + /** + * Test not. (Based on an idea by Ciaran Jessup) + */ + @Test + public void testNot() { + System.out.println("testing not"); + EWAHCompressedBitmap ewah = new EWAHCompressedBitmap(); + for (int i = 0; i <= 184; ++i) { + ewah.set(i); + } + Assert.assertEquals(ewah.cardinality(), 185); + ewah.not(); + Assert.assertEquals(ewah.cardinality(), 0); + } + + @Test + public void testOrCardinality() { + System.out.println("testing Or Cardinality"); + for (int N = 0; N < 1024; ++N) { + EWAHCompressedBitmap bitmap = new EWAHCompressedBitmap(); + for (int i = 0; i < N; i++) { + bitmap.set(i); + } + bitmap.set(1025); + bitmap.set(1026); + Assert.assertEquals(N + 2, bitmap.cardinality()); + EWAHCompressedBitmap orbitmap = bitmap.or(bitmap); + assertEquals(orbitmap, bitmap); + Assert.assertEquals(N + 2, orbitmap.cardinality()); + + Assert.assertEquals(N + 2, + bitmap.orCardinality(new EWAHCompressedBitmap())); + } + } + + /** + * Test sets and gets. + */ + @Test + public void testSetGet() { + System.out.println("testing EWAH set/get"); + EWAHCompressedBitmap ewcb = new EWAHCompressedBitmap(); + int[] val = { 5, 4400, 44600, 55400, 1000000 }; + for (int k = 0; k < val.length; ++k) { + ewcb.set(val[k]); + } + List result = ewcb.getPositions(); + Assert.assertTrue(val.length == result.size()); + for (int k = 0; k < val.length; ++k) { + Assert.assertEquals(result.get(k).intValue(), val[k]); + } + } + + @Test + public void testHashCode() { + System.out.println("testing hashCode"); + EWAHCompressedBitmap ewcb = EWAHCompressedBitmap.bitmapOf(50, 70).and( + EWAHCompressedBitmap.bitmapOf(50, 1000)); + Assert.assertEquals(EWAHCompressedBitmap.bitmapOf(50), ewcb); + Assert.assertEquals(EWAHCompressedBitmap.bitmapOf(50).hashCode(), + ewcb.hashCode()); + } + + @Test + public void testSetSizeInBits() { + System.out.println("testing SetSizeInBits"); + testSetSizeInBits(130, 131); + testSetSizeInBits(63, 64); + testSetSizeInBits(64, 65); + testSetSizeInBits(64, 128); + testSetSizeInBits(35, 131); + testSetSizeInBits(130, 400); + testSetSizeInBits(130, 191); + testSetSizeInBits(130, 192); + EWAHCompressedBitmap bitmap = new EWAHCompressedBitmap(); + bitmap.set(31); + bitmap.setSizeInBits(130, false); + bitmap.set(131); + BitSet jdkBitmap = new BitSet(); + jdkBitmap.set(31); + jdkBitmap.set(131); + assertEquals(jdkBitmap, bitmap); + } + + /** + * Test with parameters. + * + * @throws IOException + * Signals that an I/O exception has occurred. + */ + @Test + public void testWithParameters() throws IOException { + System.out + .println("These tests can run for several minutes. Please be patient."); + for (int k = 2; k < 1 << 24; k *= 8) + shouldSetBits(k); + PolizziTest(64); + PolizziTest(128); + PolizziTest(256); + PolizziTest(2048); + System.out.println("Your code is probably ok."); + } + + /** + * Pseudo-non-deterministic test inspired by S.J.vanSchaik. (Yes, + * non-deterministic tests are bad, but the test is actually deterministic.) + */ + @Test + public void vanSchaikTest() { + System.out.println("testing vanSchaikTest (this takes some time)"); + final int totalNumBits = 32768; + final double odds = 0.9; + Random rand = new Random(323232323); + for (int t = 0; t < 100; t++) { + int numBitsSet = 0; + EWAHCompressedBitmap cBitMap = new EWAHCompressedBitmap(); + for (int i = 0; i < totalNumBits; i++) { + if (rand.nextDouble() < odds) { + cBitMap.set(i); + numBitsSet++; + } + } + Assert.assertEquals(cBitMap.cardinality(), numBitsSet); + } + + } + + /** + * Function used in a test inspired by Federico Fissore. + * + * @param size + * the number of set bits + * @param seed + * the random seed + * @return the pseudo-random array int[] + */ + public static int[] createSortedIntArrayOfBitsToSet(int size, int seed) { + Random random = new Random(seed); + // build raw int array + int[] bits = new int[size]; + for (int i = 0; i < bits.length; i++) { + bits[i] = random.nextInt(TEST_BS_SIZE); + } + // might generate duplicates + Arrays.sort(bits); + // first count how many distinct values + int counter = 0; + int oldx = -1; + for (int x : bits) { + if (x != oldx) + ++counter; + oldx = x; + } + // then construct new array + int[] answer = new int[counter]; + counter = 0; + oldx = -1; + for (int x : bits) { + if (x != oldx) { + answer[counter] = x; + ++counter; + } + oldx = x; + } + return answer; + } + + /** + * Test inspired by Bilal Tayara + */ + @Test + public void TayaraTest() { + System.out.println("Tayara test"); + for (int offset = 64; offset < (1 << 30); offset *= 2) { + EWAHCompressedBitmap a = new EWAHCompressedBitmap(); + EWAHCompressedBitmap b = new EWAHCompressedBitmap(); + for (int k = 0; k < 64; ++k) { + a.set(offset + k); + b.set(offset + k); + } + if (!a.and(b).equals(a)) + throw new RuntimeException("bug"); + if (!a.or(b).equals(a)) + throw new RuntimeException("bug"); + } + } + + @Test public void TestCloneEwahCompressedBitArray() throws CloneNotSupportedException { System.out.println("testing EWAH clone"); @@ -910,7 +1024,7 @@ EWAHCompressedBitmap b; - b = (EWAHCompressedBitmap) a.clone(); + b = a.clone(); a.setSizeInBits(487123, false); b.setSizeInBits(487123, false); @@ -918,375 +1032,419 @@ Assert.assertTrue(a.equals(b)); } - /** - * a non-deterministic test proposed by Marc Polizzi. - * - * @param maxlength the maximum uncompressed size of the bitmap - */ - public static void PolizziTest(int maxlength) { - System.out.println("Polizzi test with max length = " + maxlength); - for (int k = 0; k < 10000; ++k) { - final Random rnd = new Random(); - final EWAHCompressedBitmap ewahBitmap1 = new EWAHCompressedBitmap(); - final BitSet jdkBitmap1 = new BitSet(); - final EWAHCompressedBitmap ewahBitmap2 = new EWAHCompressedBitmap(); - final BitSet jdkBitmap2 = new BitSet(); - final EWAHCompressedBitmap ewahBitmap3 = new EWAHCompressedBitmap(); - final BitSet jdkBitmap3 = new BitSet(); - final int len = rnd.nextInt(maxlength); - for (int pos = 0; pos < len; pos++) { // random *** number of bits set *** - if (rnd.nextInt(7) == 0) { // random *** increasing *** values - ewahBitmap1.set(pos); - jdkBitmap1.set(pos); - } - if (rnd.nextInt(11) == 0) { // random *** increasing *** values - ewahBitmap2.set(pos); - jdkBitmap2.set(pos); - } - if (rnd.nextInt(7) == 0) { // random *** increasing *** values - ewahBitmap3.set(pos); - jdkBitmap3.set(pos); - } - } - assertEquals(jdkBitmap1, ewahBitmap1); - assertEquals(jdkBitmap2, ewahBitmap2); - assertEquals(jdkBitmap3, ewahBitmap3); - // XOR - { - final EWAHCompressedBitmap xorEwahBitmap = ewahBitmap1.xor(ewahBitmap2); - final BitSet xorJdkBitmap = (BitSet) jdkBitmap1.clone(); - xorJdkBitmap.xor(jdkBitmap2); - assertEquals(xorJdkBitmap, xorEwahBitmap); - } - // AND - { - final EWAHCompressedBitmap andEwahBitmap = ewahBitmap1.and(ewahBitmap2); - final BitSet andJdkBitmap = (BitSet) jdkBitmap1.clone(); - andJdkBitmap.and(jdkBitmap2); - assertEquals(andJdkBitmap, andEwahBitmap); - } - // AND - { - final EWAHCompressedBitmap andEwahBitmap = ewahBitmap2.and(ewahBitmap1); - final BitSet andJdkBitmap = (BitSet) jdkBitmap1.clone(); - andJdkBitmap.and(jdkBitmap2); - assertEquals(andJdkBitmap, andEwahBitmap); - assertEquals(andJdkBitmap, EWAHCompressedBitmap.and(ewahBitmap1, ewahBitmap2)); - } - // MULTI AND - { - final BitSet andJdkBitmap = (BitSet) jdkBitmap1.clone(); - andJdkBitmap.and(jdkBitmap2); - andJdkBitmap.and(jdkBitmap3); - assertEquals(andJdkBitmap, EWAHCompressedBitmap.and(ewahBitmap1, ewahBitmap2, ewahBitmap3)); - assertEquals(andJdkBitmap, EWAHCompressedBitmap.and(ewahBitmap3, ewahBitmap2, ewahBitmap1)); - Assert.assertEquals(andJdkBitmap.cardinality(), EWAHCompressedBitmap.andCardinality(ewahBitmap1, ewahBitmap2, ewahBitmap3)); - } - // AND NOT - { - final EWAHCompressedBitmap andNotEwahBitmap = ewahBitmap1 - .andNot(ewahBitmap2); - final BitSet andNotJdkBitmap = (BitSet) jdkBitmap1.clone(); - andNotJdkBitmap.andNot(jdkBitmap2); - assertEquals(andNotJdkBitmap, andNotEwahBitmap); - } - // AND NOT - { - final EWAHCompressedBitmap andNotEwahBitmap = ewahBitmap2 - .andNot(ewahBitmap1); - final BitSet andNotJdkBitmap = (BitSet) jdkBitmap2.clone(); - andNotJdkBitmap.andNot(jdkBitmap1); - assertEquals(andNotJdkBitmap, andNotEwahBitmap); - } - // OR - { - final EWAHCompressedBitmap orEwahBitmap = ewahBitmap1.or(ewahBitmap2); - final BitSet orJdkBitmap = (BitSet) jdkBitmap1.clone(); - orJdkBitmap.or(jdkBitmap2); - assertEquals(orJdkBitmap, orEwahBitmap); - assertEquals(orJdkBitmap, EWAHCompressedBitmap.or(ewahBitmap1, ewahBitmap2)); - Assert.assertEquals(orEwahBitmap.cardinality(), ewahBitmap1.orCardinality(ewahBitmap2)); - } - // OR - { - final EWAHCompressedBitmap orEwahBitmap = ewahBitmap2.or(ewahBitmap1); - final BitSet orJdkBitmap = (BitSet) jdkBitmap1.clone(); - orJdkBitmap.or(jdkBitmap2); - assertEquals(orJdkBitmap, orEwahBitmap); - } - // MULTI OR - { - final BitSet orJdkBitmap = (BitSet) jdkBitmap1.clone(); - orJdkBitmap.or(jdkBitmap2); - orJdkBitmap.or(jdkBitmap3); - assertEquals(orJdkBitmap, EWAHCompressedBitmap.or(ewahBitmap1, ewahBitmap2, ewahBitmap3)); - assertEquals(orJdkBitmap, EWAHCompressedBitmap.or(ewahBitmap3, ewahBitmap2, ewahBitmap1)); - Assert.assertEquals(orJdkBitmap.cardinality(), EWAHCompressedBitmap.orCardinality(ewahBitmap1, ewahBitmap2, ewahBitmap3)); - } - } - } - - /** - * Pseudo-non-deterministic test inspired by Federico Fissore. - * - * @param length the number of set bits in a bitmap - */ - public static void shouldSetBits(int length) { - System.out.println("testing shouldSetBits " + length); - int[] bitsToSet = createSortedIntArrayOfBitsToSet(length,434222); - EWAHCompressedBitmap ewah = new EWAHCompressedBitmap(); - System.out.println(" ... setting " + bitsToSet.length + " values"); - for (int i : bitsToSet) { - ewah.set(i); - } - System.out.println(" ... verifying " + bitsToSet.length + " values"); - equal(ewah.iterator(), bitsToSet); - System.out.println(" ... checking cardinality"); - Assert.assertEquals(bitsToSet.length, ewah.cardinality()); - } - - @Test - public void testSizeInBits1() { - EWAHCompressedBitmap bitmap = new EWAHCompressedBitmap(); - bitmap.setSizeInBits(1, false); - bitmap.not(); - Assert.assertEquals(1, bitmap.cardinality()); - } - - @Test - public void testHasNextSafe() { - EWAHCompressedBitmap bitmap = new EWAHCompressedBitmap(); - bitmap.set(0); - IntIterator it = bitmap.intIterator(); - Assert.assertTrue(it.hasNext()); - Assert.assertEquals(0, it.next()); - } - - - @Test - public void testHasNextSafe2() { - EWAHCompressedBitmap bitmap = new EWAHCompressedBitmap(); - bitmap.set(0); - IntIterator it = bitmap.intIterator(); - Assert.assertEquals(0, it.next()); - } - - - @Test - public void testInfiniteLoop() { - System.out.println("Testing for an infinite loop"); + /** + * a non-deterministic test proposed by Marc Polizzi. + * + * @param maxlength + * the maximum uncompressed size of the bitmap + */ + public static void PolizziTest(int maxlength) { + System.out.println("Polizzi test with max length = " + maxlength); + for (int k = 0; k < 10000; ++k) { + final Random rnd = new Random(); + final EWAHCompressedBitmap ewahBitmap1 = new EWAHCompressedBitmap(); + final BitSet jdkBitmap1 = new BitSet(); + final EWAHCompressedBitmap ewahBitmap2 = new EWAHCompressedBitmap(); + final BitSet jdkBitmap2 = new BitSet(); + final EWAHCompressedBitmap ewahBitmap3 = new EWAHCompressedBitmap(); + final BitSet jdkBitmap3 = new BitSet(); + final int len = rnd.nextInt(maxlength); + for (int pos = 0; pos < len; pos++) { // random *** number of bits + // set *** + if (rnd.nextInt(7) == 0) { // random *** increasing *** values + ewahBitmap1.set(pos); + jdkBitmap1.set(pos); + } + if (rnd.nextInt(11) == 0) { // random *** increasing *** values + ewahBitmap2.set(pos); + jdkBitmap2.set(pos); + } + if (rnd.nextInt(7) == 0) { // random *** increasing *** values + ewahBitmap3.set(pos); + jdkBitmap3.set(pos); + } + } + assertEquals(jdkBitmap1, ewahBitmap1); + assertEquals(jdkBitmap2, ewahBitmap2); + assertEquals(jdkBitmap3, ewahBitmap3); + // XOR + { + final EWAHCompressedBitmap xorEwahBitmap = ewahBitmap1 + .xor(ewahBitmap2); + final BitSet xorJdkBitmap = (BitSet) jdkBitmap1.clone(); + xorJdkBitmap.xor(jdkBitmap2); + assertEquals(xorJdkBitmap, xorEwahBitmap); + } + // AND + { + final EWAHCompressedBitmap andEwahBitmap = ewahBitmap1 + .and(ewahBitmap2); + final BitSet andJdkBitmap = (BitSet) jdkBitmap1.clone(); + andJdkBitmap.and(jdkBitmap2); + assertEquals(andJdkBitmap, andEwahBitmap); + } + // AND + { + final EWAHCompressedBitmap andEwahBitmap = ewahBitmap2 + .and(ewahBitmap1); + final BitSet andJdkBitmap = (BitSet) jdkBitmap1.clone(); + andJdkBitmap.and(jdkBitmap2); + assertEquals(andJdkBitmap, andEwahBitmap); + assertEquals(andJdkBitmap, + EWAHCompressedBitmap.and(ewahBitmap1, ewahBitmap2)); + } + // MULTI AND + { + final BitSet andJdkBitmap = (BitSet) jdkBitmap1.clone(); + andJdkBitmap.and(jdkBitmap2); + andJdkBitmap.and(jdkBitmap3); + assertEquals(andJdkBitmap, EWAHCompressedBitmap.and( + ewahBitmap1, ewahBitmap2, ewahBitmap3)); + assertEquals(andJdkBitmap, EWAHCompressedBitmap.and( + ewahBitmap3, ewahBitmap2, ewahBitmap1)); + Assert.assertEquals(andJdkBitmap.cardinality(), + EWAHCompressedBitmap.andCardinality(ewahBitmap1, + ewahBitmap2, ewahBitmap3)); + } + // AND NOT + { + final EWAHCompressedBitmap andNotEwahBitmap = ewahBitmap1 + .andNot(ewahBitmap2); + final BitSet andNotJdkBitmap = (BitSet) jdkBitmap1.clone(); + andNotJdkBitmap.andNot(jdkBitmap2); + assertEquals(andNotJdkBitmap, andNotEwahBitmap); + } + // AND NOT + { + final EWAHCompressedBitmap andNotEwahBitmap = ewahBitmap2 + .andNot(ewahBitmap1); + final BitSet andNotJdkBitmap = (BitSet) jdkBitmap2.clone(); + andNotJdkBitmap.andNot(jdkBitmap1); + assertEquals(andNotJdkBitmap, andNotEwahBitmap); + } + // OR + { + final EWAHCompressedBitmap orEwahBitmap = ewahBitmap1 + .or(ewahBitmap2); + final BitSet orJdkBitmap = (BitSet) jdkBitmap1.clone(); + orJdkBitmap.or(jdkBitmap2); + assertEquals(orJdkBitmap, orEwahBitmap); + assertEquals(orJdkBitmap, + EWAHCompressedBitmap.or(ewahBitmap1, ewahBitmap2)); + Assert.assertEquals(orEwahBitmap.cardinality(), + ewahBitmap1.orCardinality(ewahBitmap2)); + } + // OR + { + final EWAHCompressedBitmap orEwahBitmap = ewahBitmap2 + .or(ewahBitmap1); + final BitSet orJdkBitmap = (BitSet) jdkBitmap1.clone(); + orJdkBitmap.or(jdkBitmap2); + assertEquals(orJdkBitmap, orEwahBitmap); + } + // MULTI OR + { + final BitSet orJdkBitmap = (BitSet) jdkBitmap1.clone(); + orJdkBitmap.or(jdkBitmap2); + orJdkBitmap.or(jdkBitmap3); + assertEquals(orJdkBitmap, EWAHCompressedBitmap.or(ewahBitmap1, + ewahBitmap2, ewahBitmap3)); + assertEquals(orJdkBitmap, EWAHCompressedBitmap.or(ewahBitmap3, + ewahBitmap2, ewahBitmap1)); + Assert.assertEquals(orJdkBitmap.cardinality(), + EWAHCompressedBitmap.orCardinality(ewahBitmap1, + ewahBitmap2, ewahBitmap3)); + } + } + } + + /** + * Pseudo-non-deterministic test inspired by Federico Fissore. + * + * @param length + * the number of set bits in a bitmap + */ + public static void shouldSetBits(int length) { + System.out.println("testing shouldSetBits " + length); + int[] bitsToSet = createSortedIntArrayOfBitsToSet(length, 434222); + EWAHCompressedBitmap ewah = new EWAHCompressedBitmap(); + System.out.println(" ... setting " + bitsToSet.length + " values"); + for (int i : bitsToSet) { + ewah.set(i); + } + System.out.println(" ... verifying " + bitsToSet.length + " values"); + equal(ewah.iterator(), bitsToSet); + System.out.println(" ... checking cardinality"); + Assert.assertEquals(bitsToSet.length, ewah.cardinality()); + } + + @Test + public void testSizeInBits1() { + EWAHCompressedBitmap bitmap = new EWAHCompressedBitmap(); + bitmap.setSizeInBits(1, false); + bitmap.not(); + Assert.assertEquals(1, bitmap.cardinality()); + } + + @Test + public void testHasNextSafe() { + EWAHCompressedBitmap bitmap = new EWAHCompressedBitmap(); + bitmap.set(0); + IntIterator it = bitmap.intIterator(); + Assert.assertTrue(it.hasNext()); + Assert.assertEquals(0, it.next()); + } + + @Test + public void testHasNextSafe2() { + EWAHCompressedBitmap bitmap = new EWAHCompressedBitmap(); + bitmap.set(0); + IntIterator it = bitmap.intIterator(); + Assert.assertEquals(0, it.next()); + } + + @Test + public void testInfiniteLoop() { + System.out.println("Testing for an infinite loop"); EWAHCompressedBitmap b1 = new EWAHCompressedBitmap(); EWAHCompressedBitmap b2 = new EWAHCompressedBitmap(); EWAHCompressedBitmap b3 = new EWAHCompressedBitmap(); - b3.setSizeInBits(5); + b3.setSizeInBits(5,false); b1.set(2); b2.set(4); EWAHCompressedBitmap.and(b1, b2, b3); EWAHCompressedBitmap.or(b1, b2, b3); - } + } + + @Test + public void testSizeInBits2() { + EWAHCompressedBitmap bitmap = new EWAHCompressedBitmap(); + bitmap.setSizeInBits(1, true); + bitmap.not(); + Assert.assertEquals(0, bitmap.cardinality()); + } + + private static void assertAndEquals(EWAHCompressedBitmap... bitmaps) { + EWAHCompressedBitmap expected = bitmaps[0]; + for (int i = 1; i < bitmaps.length; i++) { + expected = expected.and(bitmaps[i]); + } + Assert.assertTrue(expected.equals(EWAHCompressedBitmap.and(bitmaps))); + } + + private static void assertEquals(EWAHCompressedBitmap expected, + EWAHCompressedBitmap actual) { + Assert.assertEquals(expected.sizeInBits(), actual.sizeInBits()); + assertEqualsPositions(expected, actual); + } + + private static void assertOrEquals(EWAHCompressedBitmap... bitmaps) { + EWAHCompressedBitmap expected = bitmaps[0]; + for (int i = 1; i < bitmaps.length; i++) { + expected = expected.or(bitmaps[i]); + } + assertEquals(expected, EWAHCompressedBitmap.or(bitmaps)); + } + + /** + * Extracted. + * + * @param bits + * the bits + * @return the integer + */ + private static Integer extracted(final Iterator bits) { + return bits.next(); + } + + private static void testSetSizeInBits(int size, int nextBit) { + EWAHCompressedBitmap bitmap = new EWAHCompressedBitmap(); + bitmap.setSizeInBits(size, false); + bitmap.set(nextBit); + BitSet jdkBitmap = new BitSet(); + jdkBitmap.set(nextBit); + assertEquals(jdkBitmap, bitmap); + } + + /** + * Assess equality between an uncompressed bitmap and a compressed one, part + * of a test contributed by Marc Polizzi + * + * @param jdkBitmap + * the uncompressed bitmap + * @param ewahBitmap + * the compressed bitmap + */ + static void assertCardinality(BitSet jdkBitmap, + EWAHCompressedBitmap ewahBitmap) { + final int c1 = jdkBitmap.cardinality(); + final int c2 = ewahBitmap.cardinality(); + Assert.assertEquals(c1, c2); + } + + /** + * Assess equality between an uncompressed bitmap and a compressed one, part + * of a test contributed by Marc Polizzi. + * + * @param jdkBitmap + * the uncompressed bitmap + * @param ewahBitmap + * the compressed bitmap + */ + static void assertEquals(BitSet jdkBitmap, EWAHCompressedBitmap ewahBitmap) { + assertEqualsIterator(jdkBitmap, ewahBitmap); + assertEqualsPositions(jdkBitmap, ewahBitmap); + assertCardinality(jdkBitmap, ewahBitmap); + } + + static void assertEquals(int[] v, List p) { + assertEquals(p, v); + } + + static void assertEquals(List p, int[] v) { + if (v.length != p.size()) + throw new RuntimeException("Different lengths " + v.length + " " + + p.size()); + for (int k = 0; k < v.length; ++k) + if (v[k] != p.get(k).intValue()) + throw new RuntimeException("expected equal at " + k + " " + + v[k] + " " + p.get(k)); + } + + // + /** + * Assess equality between an uncompressed bitmap and a compressed one, part + * of a test contributed by Marc Polizzi + * + * @param jdkBitmap + * the jdk bitmap + * @param ewahBitmap + * the ewah bitmap + */ + static void assertEqualsIterator(BitSet jdkBitmap, + EWAHCompressedBitmap ewahBitmap) { + final Vector positions = new Vector(); + final Iterator bits = ewahBitmap.iterator(); + while (bits.hasNext()) { + final int bit = extracted(bits).intValue(); + Assert.assertTrue(jdkBitmap.get(bit)); + positions.add(new Integer(bit)); + } + for (int pos = jdkBitmap.nextSetBit(0); pos >= 0; pos = jdkBitmap + .nextSetBit(pos + 1)) { + if (!positions.contains(new Integer(pos))) { + throw new RuntimeException( + "iterator: bitset got different bits"); + } + } + } + + // part of a test contributed by Marc Polizzi + /** + * Assert equals positions. + * + * @param jdkBitmap + * the jdk bitmap + * @param ewahBitmap + * the ewah bitmap + */ + static void assertEqualsPositions(BitSet jdkBitmap, + EWAHCompressedBitmap ewahBitmap) { + final List positions = ewahBitmap.getPositions(); + for (int position : positions) { + if (!jdkBitmap.get(position)) { + throw new RuntimeException( + "positions: bitset got different bits"); + } + } + for (int pos = jdkBitmap.nextSetBit(0); pos >= 0; pos = jdkBitmap + .nextSetBit(pos + 1)) { + if (!positions.contains(new Integer(pos))) { + throw new RuntimeException( + "positions: bitset got different bits"); + } + } + // we check again + final int[] fastpositions = ewahBitmap.toArray(); + for (int position : fastpositions) { + if (!jdkBitmap.get(position)) { + throw new RuntimeException( + "positions: bitset got different bits with toArray"); + } + } + for (int pos = jdkBitmap.nextSetBit(0); pos >= 0; pos = jdkBitmap + .nextSetBit(pos + 1)) { + int index = Arrays.binarySearch(fastpositions, pos); + if (index < 0) + throw new RuntimeException( + "positions: bitset got different bits with toArray"); + if (fastpositions[index] != pos) + throw new RuntimeException( + "positions: bitset got different bits with toArray"); + } + } + + /** + * Assert equals positions. + * + * @param ewahBitmap1 + * the ewah bitmap1 + * @param ewahBitmap2 + * the ewah bitmap2 + */ + static void assertEqualsPositions(EWAHCompressedBitmap ewahBitmap1, + EWAHCompressedBitmap ewahBitmap2) { + final List positions1 = ewahBitmap1.getPositions(); + final List positions2 = ewahBitmap2.getPositions(); + if (!positions1.equals(positions2)) + throw new RuntimeException( + "positions: alternative got different bits (two bitmaps)"); + // + final int[] fastpositions1 = ewahBitmap1.toArray(); + assertEquals(fastpositions1, positions1); + final int[] fastpositions2 = ewahBitmap2.toArray(); + assertEquals(fastpositions2, positions2); + if (!Arrays.equals(fastpositions1, fastpositions2)) + throw new RuntimeException( + "positions: alternative got different bits with toArray but not with getPositions (two bitmaps)"); + } + + /** + * Convenience function to assess equality between a compressed bitset and + * an uncompressed bitset + * + * @param x + * the compressed bitset/bitmap + * @param y + * the uncompressed bitset/bitmap + */ + static void equal(EWAHCompressedBitmap x, BitSet y) { + Assert.assertEquals(x.cardinality(), y.cardinality()); + for (int i : x.getPositions()) + Assert.assertTrue(y.get(i)); + } + + /** + * Convenience function to assess equality between an array and an iterator + * over Integers + * + * @param i + * the iterator + * @param array + * the array + */ + static void equal(Iterator i, int[] array) { + int cursor = 0; + while (i.hasNext()) { + int x = extracted(i).intValue(); + int y = array[cursor++]; + Assert.assertEquals(x, y); + } + } - @Test - public void testSizeInBits2() { - EWAHCompressedBitmap bitmap = new EWAHCompressedBitmap(); - bitmap.setSizeInBits(1, true); - bitmap.not(); - Assert.assertEquals(0, bitmap.cardinality()); - } - private static void assertAndEquals(EWAHCompressedBitmap...bitmaps) - { - EWAHCompressedBitmap expected = bitmaps[0]; - for(int i = 1; i < bitmaps.length; i++) { - expected = expected.and(bitmaps[i]); - } - assertEquals(expected, EWAHCompressedBitmap.and(bitmaps)); - } - - private static void assertEquals(EWAHCompressedBitmap expected, EWAHCompressedBitmap actual) { - Assert.assertEquals(expected.sizeInBits(), actual.sizeInBits()); - assertEqualsPositions(expected, actual); - } - - private static void assertOrEquals(EWAHCompressedBitmap...bitmaps) - { - EWAHCompressedBitmap expected = bitmaps[0]; - for(int i = 1; i < bitmaps.length; i++) { - expected = expected.or(bitmaps[i]); - } - assertEquals(expected, EWAHCompressedBitmap.or(bitmaps)); - } - - /** - * Extracted. - * - * @param bits the bits - * @return the integer - */ - private static Integer extracted(final Iterator bits) { - return bits.next(); - } - private static void testSetSizeInBits(int size, int nextBit) { - EWAHCompressedBitmap bitmap = new EWAHCompressedBitmap(); - bitmap.setSizeInBits(size,false); - bitmap.set(nextBit); - BitSet jdkBitmap = new BitSet(); - jdkBitmap.set(nextBit); - assertEquals(jdkBitmap,bitmap); - } - - /** - * Assess equality between an uncompressed bitmap and a compressed one, - * part of a test contributed by Marc Polizzi - * - * @param jdkBitmap the uncompressed bitmap - * @param ewahBitmap the compressed bitmap - */ - static void assertCardinality(BitSet jdkBitmap, - EWAHCompressedBitmap ewahBitmap) { - final int c1 = jdkBitmap.cardinality(); - final int c2 = ewahBitmap.cardinality(); - Assert.assertEquals(c1, c2); - } - - /** - * Assess equality between an uncompressed bitmap and a compressed one, - * part of a test contributed by Marc Polizzi. - * - * @param jdkBitmap the uncompressed bitmap - * @param ewahBitmap the compressed bitmap - */ - static void assertEquals(BitSet jdkBitmap, EWAHCompressedBitmap ewahBitmap) { - assertEqualsIterator(jdkBitmap, ewahBitmap); - assertEqualsPositions(jdkBitmap, ewahBitmap); - assertCardinality(jdkBitmap, ewahBitmap); - } - - static void assertEquals(int[] v, List p) { - assertEquals(p,v); - } - static void assertEquals(List p, int[] v) { - if(v.length!= p.size()) - throw new RuntimeException("Different lengths "+v.length+ " "+p.size()); - for(int k = 0; k positions = new Vector(); - final Iterator bits = ewahBitmap.iterator(); - while (bits.hasNext()) { - final int bit = extracted(bits).intValue(); - Assert.assertTrue(jdkBitmap.get(bit)); - positions.add(new Integer(bit)); - } - for (int pos = jdkBitmap.nextSetBit(0); pos >= 0; pos = jdkBitmap - .nextSetBit(pos + 1)) { - if (!positions.contains(new Integer(pos))) { - throw new RuntimeException("iterator: bitset got different bits"); - } - } - } - - // part of a test contributed by Marc Polizzi - /** - * Assert equals positions. - * - * @param jdkBitmap the jdk bitmap - * @param ewahBitmap the ewah bitmap - */ - static void assertEqualsPositions(BitSet jdkBitmap, - EWAHCompressedBitmap ewahBitmap) { - final List positions = ewahBitmap.getPositions(); - for (int position : positions) { - if (!jdkBitmap.get(position)) { - throw new RuntimeException("positions: bitset got different bits"); - } - } - for (int pos = jdkBitmap.nextSetBit(0); pos >= 0; pos = jdkBitmap - .nextSetBit(pos + 1)) { - if (!positions.contains(new Integer(pos))) { - throw new RuntimeException("positions: bitset got different bits"); - } - } - // we check again - final int[] fastpositions = ewahBitmap.toArray(); - for (int position : fastpositions) { - if (!jdkBitmap.get(position)) { - throw new RuntimeException("positions: bitset got different bits with toArray"); - } - } - for (int pos = jdkBitmap.nextSetBit(0); pos >= 0; pos = jdkBitmap - .nextSetBit(pos + 1)) { - int index = Arrays.binarySearch(fastpositions,pos); - if(index < 0) - throw new RuntimeException("positions: bitset got different bits with toArray"); - if(fastpositions[index]!= pos) - throw new RuntimeException("positions: bitset got different bits with toArray"); - } - } - - /** - * Assert equals positions. - * - * @param ewahBitmap1 the ewah bitmap1 - * @param ewahBitmap2 the ewah bitmap2 - */ - static void assertEqualsPositions(EWAHCompressedBitmap ewahBitmap1, - EWAHCompressedBitmap ewahBitmap2) { - final List positions1 = ewahBitmap1.getPositions(); - final List positions2 = ewahBitmap2.getPositions(); - if (!positions1.equals(positions2)) - throw new RuntimeException("positions: alternative got different bits (two bitmaps)"); - // - final int[] fastpositions1 = ewahBitmap1.toArray(); - assertEquals(fastpositions1, positions1); - final int[] fastpositions2 = ewahBitmap2.toArray(); - assertEquals(fastpositions2, positions2); - if (!Arrays.equals(fastpositions1, fastpositions2)) - throw new RuntimeException("positions: alternative got different bits with toArray but not with getPositions (two bitmaps)"); - } - - /** - * Convenience function to assess equality between a compressed bitset - * and an uncompressed bitset - * - * @param x the compressed bitset/bitmap - * @param y the uncompressed bitset/bitmap - */ - static void equal(EWAHCompressedBitmap x, BitSet y) { - Assert.assertEquals (x.cardinality() , y.cardinality()); - for (int i : x.getPositions()) - Assert.assertTrue(y.get(i)); - } - - /** - * Convenience function to assess equality between an array and an iterator over - * Integers - * - * @param i the iterator - * @param array the array - */ - static void equal(Iterator i, int[] array) { - int cursor = 0; - while (i.hasNext()) { - int x = extracted(i).intValue(); - int y = array[cursor++]; - Assert.assertEquals(x,y); - } - } - - /** The Constant MEGA: a large integer. */ - private static final int MEGA = 8 * 1024 * 1024; + /** The Constant MEGA: a large integer. */ + private static final int MEGA = 8 * 1024 * 1024; - /** The Constant TEST_BS_SIZE: used to represent the size of a large bitmap. */ - private static final int TEST_BS_SIZE = 8 * MEGA; + /** The Constant TEST_BS_SIZE: used to represent the size of a large bitmap. */ + private static final int TEST_BS_SIZE = 8 * MEGA; } diff -Nru libjavaewah-java-0.6.12/src/test/java/com/googlecode/javaewah/IntIteratorOverIteratingRLWTest.java libjavaewah-java-0.7.9/src/test/java/com/googlecode/javaewah/IntIteratorOverIteratingRLWTest.java --- libjavaewah-java-0.6.12/src/test/java/com/googlecode/javaewah/IntIteratorOverIteratingRLWTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libjavaewah-java-0.7.9/src/test/java/com/googlecode/javaewah/IntIteratorOverIteratingRLWTest.java 2013-11-12 14:31:20.000000000 +0000 @@ -0,0 +1,116 @@ +package com.googlecode.javaewah; + +import static org.junit.Assert.*; +import org.junit.Test; + +/* + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. + */ +/** + * Tests for utility class. Sketchy for now. + * + */ +@SuppressWarnings("javadoc") +public class IntIteratorOverIteratingRLWTest { + + @Test + // had problems with bitmaps beginning with two consecutive clean runs + public void testConsecClean() { + System.out + .println("testing int iteration, 2 consec clean runs starting with zeros"); + EWAHCompressedBitmap e = new EWAHCompressedBitmap(); + for (int i = 64; i < 128; ++i) + e.set(i); + IntIteratorOverIteratingRLW ii = new IntIteratorOverIteratingRLW( + e.getIteratingRLW()); + assertTrue(ii.hasNext()); + int ctr = 0; + while (ii.hasNext()) { + ++ctr; + ii.next(); + } + assertEquals(64, ctr); + } + + @Test + public void testConsecCleanStartOnes() { + System.out + .println("testing int iteration, 2 consec clean runs starting with ones"); + EWAHCompressedBitmap e = new EWAHCompressedBitmap(); + for (int i = 0; i < 2 * 64; ++i) + e.set(i); + for (int i = 4 * 64; i < 5 * 64; ++i) + e.set(i); + + IntIteratorOverIteratingRLW ii = new IntIteratorOverIteratingRLW( + e.getIteratingRLW()); + assertTrue(ii.hasNext()); + int ctr = 0; + while (ii.hasNext()) { + ++ctr; + ii.next(); + } + assertEquals(3 * 64, ctr); + } + + @Test + public void testStartDirty() { + System.out.println("testing int iteration, no initial runs"); + EWAHCompressedBitmap e = new EWAHCompressedBitmap(); + for (int i = 1; i < 2 * 64; ++i) + e.set(i); + for (int i = 4 * 64; i < 5 * 64; ++i) + e.set(i); + + IntIteratorOverIteratingRLW ii = new IntIteratorOverIteratingRLW( + e.getIteratingRLW()); + assertTrue(ii.hasNext()); + int ctr = 0; + while (ii.hasNext()) { + ++ctr; + ii.next(); + } + assertEquals(3 * 64 - 1, ctr); + } + + @Test + public void testEmpty() { + System.out.println("testing int iteration over empty bitmap"); + EWAHCompressedBitmap e = new EWAHCompressedBitmap(); + + IntIteratorOverIteratingRLW ii = new IntIteratorOverIteratingRLW( + e.getIteratingRLW()); + assertFalse(ii.hasNext()); + } + + @Test + public void testRandomish() { + EWAHCompressedBitmap e = new EWAHCompressedBitmap(); + + int upperlimit = 100000; + for (int i = 0; i < upperlimit; ++i) { + double probabilityOfOne = i / (double) (upperlimit / 2); + if (probabilityOfOne > 1.0) + probabilityOfOne = 1.0; + if (Math.random() < probabilityOfOne) { + e.set(i); + } + } + + IntIteratorOverIteratingRLW ii = new IntIteratorOverIteratingRLW( + e.getIteratingRLW()); + int ctr = 0; + while (ii.hasNext()) { + ++ctr; + ii.next(); + } + + assertEquals(e.cardinality(), ctr); + System.out + .println("checking int iteration over a var density bitset of size " + + e.cardinality()); + + } + +} diff -Nru libjavaewah-java-0.6.12/src/test/java/com/googlecode/javaewah/IteratorAggregationTest.java libjavaewah-java-0.7.9/src/test/java/com/googlecode/javaewah/IteratorAggregationTest.java --- libjavaewah-java-0.6.12/src/test/java/com/googlecode/javaewah/IteratorAggregationTest.java 1970-01-01 00:00:00.000000000 +0000 +++ libjavaewah-java-0.7.9/src/test/java/com/googlecode/javaewah/IteratorAggregationTest.java 2013-11-12 14:31:20.000000000 +0000 @@ -0,0 +1,144 @@ +package com.googlecode.javaewah; + +import static org.junit.Assert.*; +import java.util.Iterator; +import org.junit.Test; +import com.googlecode.javaewah.benchmark.ClusteredDataGenerator; +/* + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. + */ +/** + * Tests specifically for iterators. + * + */ +public class IteratorAggregationTest { + + /** + * @param N Number of bitmaps to generate in each set + * @param nbr parameter determining the size of the arrays (in a log scale) + * @return an iterator over sets of bitmaps + */ + public static Iterator getCollections(final int N, final int nbr) { + final ClusteredDataGenerator cdg = new ClusteredDataGenerator(123); + return new Iterator() { + int sparsity = 1; + + @Override + public boolean hasNext() { + return this.sparsity < 5; + } + + @Override + public EWAHCompressedBitmap[] next() { + int[][] data = new int[N][]; + int Max = (1 << (nbr + this.sparsity)); + for (int k = 0; k < N; ++k) + data[k] = cdg.generateClustered(1 << nbr, Max); + EWAHCompressedBitmap[] ewah = new EWAHCompressedBitmap[N]; + for (int k = 0; k < N; ++k) { + ewah[k] = new EWAHCompressedBitmap(); + for (int x = 0; x < data[k].length; ++x) { + ewah[k].set(data[k][x]); + } + data[k] = null; + } + this.sparsity += 3; + return ewah; + } + + @Override + public void remove() { + // unimplemented + } + + }; + + } + + /** + * + */ + @Test + public void testAnd() { + for (int N = 1; N < 10; ++N) { + System.out.println("testAnd N = " + N); + Iterator i = getCollections(N,3); + while (i.hasNext()) { + EWAHCompressedBitmap[] x = i.next(); + EWAHCompressedBitmap tanswer = EWAHCompressedBitmap.and(x); + EWAHCompressedBitmap x1 = IteratorUtil + .materialize(IteratorAggregation.bufferedand(IteratorUtil + .toIterators(x))); + assertTrue(x1.equals(tanswer)); + } + System.gc(); + } + + } + + /** + * + */ + @Test + public void testOr() { + for (int N = 1; N < 10; ++N) { + System.out.println("testOr N = " + N); + Iterator i = getCollections(N,3); + while (i.hasNext()) { + EWAHCompressedBitmap[] x = i.next(); + EWAHCompressedBitmap tanswer = EWAHCompressedBitmap.or(x); + EWAHCompressedBitmap x1 = IteratorUtil + .materialize(IteratorAggregation.bufferedor(IteratorUtil + .toIterators(x))); + assertTrue(x1.equals(tanswer)); + } + System.gc(); + } + } + + /** + * + */ + @SuppressWarnings("deprecation") + @Test + public void testWideOr() { + for (int nbr = 3; nbr <= 24; nbr += 3) { + for (int N = 100; N < 1000; N += 100) { + System.out.println("testWideOr N = " + N); + Iterator i = getCollections(N, 3); + while (i.hasNext()) { + EWAHCompressedBitmap[] x = i.next(); + EWAHCompressedBitmap tanswer = EWAHCompressedBitmap.or(x); + EWAHCompressedBitmap container = new EWAHCompressedBitmap(); + FastAggregation.legacy_orWithContainer(container, x); + assertTrue(container.equals(tanswer)); + EWAHCompressedBitmap x1 = IteratorUtil + .materialize(IteratorAggregation + .bufferedor(IteratorUtil.toIterators(x))); + assertTrue(x1.equals(tanswer)); + } + System.gc(); + } + } + } + + /** + * + */ + @Test + public void testXor() { + System.out.println("testXor "); + Iterator i = getCollections(2,3); + while (i.hasNext()) { + EWAHCompressedBitmap[] x = i.next(); + EWAHCompressedBitmap tanswer = x[0].xor(x[1]); + EWAHCompressedBitmap x1 = IteratorUtil + .materialize(IteratorAggregation.bufferedxor( + x[0].getIteratingRLW(), x[1].getIteratingRLW())); + assertTrue(x1.equals(tanswer)); + } + System.gc(); + } + +} diff -Nru libjavaewah-java-0.6.12/src/test/java/com/googlecode/javaewah32/EWAHCompressedBitmap32Test.java libjavaewah-java-0.7.9/src/test/java/com/googlecode/javaewah32/EWAHCompressedBitmap32Test.java --- libjavaewah-java-0.6.12/src/test/java/com/googlecode/javaewah32/EWAHCompressedBitmap32Test.java 2013-06-11 17:48:37.000000000 +0000 +++ libjavaewah-java-0.7.9/src/test/java/com/googlecode/javaewah32/EWAHCompressedBitmap32Test.java 2013-11-12 14:31:20.000000000 +0000 @@ -1,910 +1,1029 @@ package com.googlecode.javaewah32; /* - * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc. and Veronika Zenz - * Licensed under APL 2.0. + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. */ import org.junit.Test; + +import com.googlecode.javaewah.FastAggregation; import com.googlecode.javaewah.IntIterator; import java.util.*; import java.io.*; - import junit.framework.Assert; +/** + * This class is used for basic unit testing. + */ +@SuppressWarnings("javadoc") public class EWAHCompressedBitmap32Test { - @Test - public void testDebugSetSizeInBitsTest() { - System.out.println("testing DebugSetSizeInBits"); - EWAHCompressedBitmap32 b = new EWAHCompressedBitmap32(); - - b.set(4); - - b.setSizeInBits(6, true); - - List positions = b.getPositions(); - - Assert.assertEquals(2, positions.size()); - Assert.assertEquals(Integer.valueOf(4), positions.get(0)); - Assert.assertEquals(Integer.valueOf(5), positions.get(1)); - - Iterator iterator = b.iterator(); - Assert.assertTrue(iterator.hasNext()); - Assert.assertEquals(Integer.valueOf(4), iterator.next()); - Assert.assertTrue(iterator.hasNext()); - Assert.assertEquals(Integer.valueOf(5), iterator.next()); - Assert.assertFalse(iterator.hasNext()); - - IntIterator intIterator = b.intIterator(); - Assert.assertTrue(intIterator.hasNext()); - Assert.assertEquals(4, intIterator.next()); - Assert.assertTrue(intIterator.hasNext()); - Assert.assertEquals(5, intIterator.next()); - Assert.assertFalse(intIterator.hasNext()); - - } - /** - * Created: 2/4/11 6:03 PM By: Arnon Moscona. - */ - @Test - public void EwahIteratorProblem() { - System.out.println("testing ArnonMoscona"); - EWAHCompressedBitmap32 bitmap = new EWAHCompressedBitmap32(); - for (int i = 9434560; i <= 9435159; i++) { - bitmap.set(i); - } - IntIterator iterator = bitmap.intIterator(); - List v = bitmap.getPositions(); - int[] array = bitmap.toArray(); - for (int k = 0; k < v.size(); ++k) { - Assert.assertTrue(array[k] == v.get(k).intValue()); - Assert.assertTrue(iterator.hasNext()); - final int ival = iterator.next(); - final int vval = v.get(k).intValue(); - Assert.assertTrue(ival == vval); - } - Assert.assertTrue(!iterator.hasNext()); - // - for (int k = 2; k <= 1024; k *= 2) { - int[] bitsToSet = createSortedIntArrayOfBitsToSet(k, 434455 + 5 * k); - EWAHCompressedBitmap32 ewah = new EWAHCompressedBitmap32(); - for (int i : bitsToSet) { - ewah.set(i); - } - equal(ewah.iterator(), bitsToSet); - } - } - - - /** - * Test submitted by Gregory Ssi-Yan-Kai - */ - @Test - public void SsiYanKaiTest() { - System.out.println("testing SsiYanKaiTest"); - EWAHCompressedBitmap32 a = EWAHCompressedBitmap32.bitmapOf(39935, 39936, 39937, 39938, 39939, 39940, 39941, 39942, 39943, 39944, 39945, 39946, 39947, 39948, 39949, 39950, 39951, 39952, 39953, 39954, 39955, 39956, 39957, 39958, 39959, 39960, 39961, 39962, 39963, 39964, 39965, 39966, 39967, 39968, 39969, 39970, 39971, 39972, 39973, 39974, 39975, 39976, 39977, 39978, 39979, 39980, 39981, 39982, 39983, 39984, 39985, 39986, 39987, 39988, 39989, 39990, 39991, 39992, 39993, 39994, 39995, 39996, 39997, 39998, 39999, 40000, 40001, 40002, 40003, 40004, 40005, 40006, 40007, 40008, 40009, 40010, 40011, 40012, 40013, 40014, 40015, 40016, 40017, 40018, 40019, 40020, 40021, 40022, 40023, 40024, 40025, 40026, 40027, 40028, 40029, 40030, 40031, 40032, 40033, 40034, 40035, 40036, 40037, 40038, 40039, 40040, 40041, 40042, 40043, 40044, 40045, 40046, 40047, 40048, 40049, 40050, 40051, 40052, 40053, 40054, 40055, 40056, 40057, 40058, 40059, 40060, 40061, 40062, 40063, 40064, 40065, 40066, 40067, 40068, 40069, 40070, 40071, 40072, 40073, 40074, 40075, 40076, 40077, 40078, 40079, 40080, 40081, 40082, 40083, 40084, 40085, 40086, 40087, 40088, 40089, 40090, 40091, 40092, 40093, 40094, 40095, 40096, 40097, 40098, 40099, 40100); - EWAHCompressedBitmap32 b = EWAHCompressedBitmap32.bitmapOf(39935, 39936, 39937, 39938, 39939, 39940, 39941, 39942, 39943, 39944, 39945, 39946, 39947, 39948, 39949, 39950, 39951, 39952, 39953, 39954, 39955, 39956, 39957, 39958, 39959, 39960, 39961, 39962, 39963, 39964, 39965, 39966, 39967, 39968, 39969, 39970, 39971, 39972, 39973, 39974, 39975, 39976, 39977, 39978, 39979, 39980, 39981, 39982, 39983, 39984, 39985, 39986, 39987, 39988, 39989, 39990, 39991, 39992, 39993, 39994, 39995, 39996, 39997, 39998, 39999, 270000); - LinkedHashSet aPositions = new LinkedHashSet(a.getPositions()); - int intersection = 0; - EWAHCompressedBitmap32 inter = new EWAHCompressedBitmap32(); - LinkedHashSet bPositions = new LinkedHashSet(b.getPositions()); - for (Integer integer : bPositions) { - if (aPositions.contains(integer)) { - inter.set(integer.intValue()); - ++intersection; - } - } - EWAHCompressedBitmap32 and2 = a.and(b); - if(!and2.equals(inter)) throw new RuntimeException("intersections don't match"); - if(intersection != and2.cardinality()) throw new RuntimeException("cardinalities don't match"); - } - - /** - * Test inspired by William Habermaas. - */ - @Test - public void habermaasTest() { - System.out.println("testing habermaasTest"); - BitSet bitsetaa = new BitSet(); - EWAHCompressedBitmap32 aa = new EWAHCompressedBitmap32(); - int[] val = { 55400, 1000000, 1000128 }; - for (int k = 0; k < val.length; ++k) { - aa.set(val[k]); - bitsetaa.set(val[k]); - } - equal(aa, bitsetaa); - BitSet bitsetab = new BitSet(); - EWAHCompressedBitmap32 ab = new EWAHCompressedBitmap32(); - for (int i = 4096; i < (4096 + 5); i++) { - ab.set(i); - bitsetab.set(i); - } - ab.set(99000); - bitsetab.set(99000); - ab.set(1000130); - bitsetab.set(1000130); - equal(ab, bitsetab); - EWAHCompressedBitmap32 bb = aa.or(ab); - EWAHCompressedBitmap32 bbAnd = aa.and(ab); - try { - EWAHCompressedBitmap32 abnot = (EWAHCompressedBitmap32)ab.clone(); - abnot.not(); - EWAHCompressedBitmap32 bbAnd2 = aa.andNot(abnot); - assertEquals(bbAnd2,bbAnd); - } catch (CloneNotSupportedException e) { - e.printStackTrace(); - } - BitSet bitsetbb = (BitSet) bitsetaa.clone(); - bitsetbb.or(bitsetab); - BitSet bitsetbbAnd = (BitSet) bitsetaa.clone(); - bitsetbbAnd.and(bitsetab); - equal(bbAnd, bitsetbbAnd); - equal(bb, bitsetbb); - } - - @Test - public void testAndResultAppend() { - System.out.println("testing AndResultAppend"); - EWAHCompressedBitmap32 bitmap1 = new EWAHCompressedBitmap32(); - bitmap1.set(35); - EWAHCompressedBitmap32 bitmap2 = new EWAHCompressedBitmap32(); - bitmap2.set(35); - bitmap2.set(130); - - EWAHCompressedBitmap32 resultBitmap = bitmap1.and(bitmap2); - resultBitmap.set(131); - - bitmap1.set(131); - assertEquals(bitmap1, resultBitmap); - } - - /** - * Test cardinality. - */ - @Test - public void testCardinality() { - System.out.println("testing EWAH cardinality"); - EWAHCompressedBitmap32 bitmap = new EWAHCompressedBitmap32(); - bitmap.set(Integer.MAX_VALUE - 32); - // System.out.format("Total Items %d\n", bitmap.cardinality()); - Assert.assertTrue(bitmap.cardinality() == 1); - } - - /** - * Test clear function - */ - @Test - public void testClear() { - System.out.println("testing Clear"); - EWAHCompressedBitmap32 bitmap = new EWAHCompressedBitmap32(); - bitmap.set(5); - bitmap.clear(); - bitmap.set(7); - Assert.assertTrue(1 == bitmap.cardinality()); - Assert.assertTrue(1 == bitmap.getPositions().size()); - Assert.assertTrue(1 == bitmap.toArray().length); - Assert.assertTrue(7 == bitmap.getPositions().get(0).intValue()); - Assert.assertTrue(7 == bitmap.toArray()[0]); - bitmap.clear(); - bitmap.set(5000); - Assert.assertTrue(1 == bitmap.cardinality()); - Assert.assertTrue(1 == bitmap.getPositions().size()); - Assert.assertTrue(1 == bitmap.toArray().length); - Assert.assertTrue(5000 == bitmap.getPositions().get(0).intValue()); - bitmap.set(5001); - bitmap.set(5005); - bitmap.set(5100); - bitmap.set(5500); - bitmap.clear(); - bitmap.set(5); - bitmap.set(7); - bitmap.set(1000); - bitmap.set(1001); - Assert.assertTrue(4 == bitmap.cardinality()); - List positions = bitmap.getPositions(); - Assert.assertTrue(4 == positions.size()); - Assert.assertTrue(5 == positions.get(0).intValue()); - Assert.assertTrue(7 == positions.get(1).intValue()); - Assert.assertTrue(1000 == positions.get(2).intValue()); - Assert.assertTrue(1001 == positions.get(3).intValue()); - } - - /** - * Test ewah compressed bitmap. - */ - @Test - public void testEWAHCompressedBitmap() { - System.out.println("testing EWAH"); - int zero = 0; - int specialval = 1 | (1 << 4) | (1 << 31); - int notzero = ~zero; - EWAHCompressedBitmap32 myarray1 = new EWAHCompressedBitmap32(); - myarray1.add(zero); - myarray1.add(zero); - myarray1.add(zero); - myarray1.add(specialval); - myarray1.add(specialval); - myarray1.add(notzero); - myarray1.add(zero); - Assert.assertEquals(myarray1.getPositions().size(), 6 + 32); - EWAHCompressedBitmap32 myarray2 = new EWAHCompressedBitmap32(); - myarray2.add(zero); - myarray2.add(specialval); - myarray2.add(specialval); - myarray2.add(notzero); - myarray2.add(zero); - myarray2.add(zero); - myarray2.add(zero); - Assert.assertEquals(myarray2.getPositions().size(), 6 + 32); - List data1 = myarray1.getPositions(); - List data2 = myarray2.getPositions(); - Vector logicalor = new Vector(); - { - HashSet tmp = new HashSet(); - tmp.addAll(data1); - tmp.addAll(data2); - logicalor.addAll(tmp); - } - Collections.sort(logicalor); - Vector logicaland = new Vector(); - logicaland.addAll(data1); - logicaland.retainAll(data2); - Collections.sort(logicaland); - EWAHCompressedBitmap32 arrayand = myarray1.and(myarray2); - Assert.assertTrue(arrayand.getPositions().equals(logicaland)); - EWAHCompressedBitmap32 arrayor = myarray1.or(myarray2); - Assert.assertTrue(arrayor.getPositions().equals(logicalor)); - EWAHCompressedBitmap32 arrayandbis = myarray2.and(myarray1); - Assert.assertTrue(arrayandbis.getPositions().equals(logicaland)); - EWAHCompressedBitmap32 arrayorbis = myarray2.or(myarray1); - Assert.assertTrue(arrayorbis.getPositions().equals(logicalor)); - EWAHCompressedBitmap32 x = new EWAHCompressedBitmap32(); - for (Integer i : myarray1.getPositions()) { - x.set(i.intValue()); - } - Assert.assertTrue(x.getPositions().equals(myarray1.getPositions())); - x = new EWAHCompressedBitmap32(); - for (Integer i : myarray2.getPositions()) { - x.set(i.intValue()); - } - Assert.assertTrue(x.getPositions().equals(myarray2.getPositions())); - x = new EWAHCompressedBitmap32(); - for (Iterator k = myarray1.iterator(); k.hasNext();) { - x.set(extracted(k).intValue()); - } - Assert.assertTrue(x.getPositions().equals(myarray1.getPositions())); - x = new EWAHCompressedBitmap32(); - for (Iterator k = myarray2.iterator(); k.hasNext();) { - x.set(extracted(k).intValue()); - } - Assert.assertTrue(x.getPositions().equals(myarray2.getPositions())); - } - - /** - * Test externalization. - * - * @throws IOException - * Signals that an I/O exception has occurred. - */ - @Test - public void testExternalization() throws IOException { - System.out.println("testing EWAH externalization"); - EWAHCompressedBitmap32 ewcb = new EWAHCompressedBitmap32(); - int[] val = { 5, 4400, 44600, 55400, 1000000 }; - for (int k = 0; k < val.length; ++k) { - ewcb.set(val[k]); - } - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - ObjectOutputStream oo = new ObjectOutputStream(bos); - ewcb.writeExternal(oo); - oo.close(); - ewcb = null; - ewcb = new EWAHCompressedBitmap32(); - ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); - ewcb.readExternal(new ObjectInputStream(bis)); - List result = ewcb.getPositions(); - Assert.assertTrue(val.length == result.size()); - for (int k = 0; k < val.length; ++k) { - Assert.assertTrue(result.get(k).intValue() == val[k]); - } - } - - @Test - public void testExtremeRange() { - System.out.println("testing EWAH at its extreme range"); - EWAHCompressedBitmap32 myarray1 = new EWAHCompressedBitmap32(); - int N = 1024; - for (int i = 0; i < N; ++i) { - myarray1.set(Integer.MAX_VALUE - 32 - N + i); - Assert.assertTrue(myarray1.cardinality() == i+1); - int[] val = myarray1.toArray(); - Assert.assertTrue(val[0] == Integer.MAX_VALUE - 32 - N); - } - } - - /** - * Test the intersects method - */ - @Test - public void testIntersectsMethod(){ - System.out.println("testing Intersets Bug"); - EWAHCompressedBitmap32 bitmap = new EWAHCompressedBitmap32(); - bitmap.set(1); - EWAHCompressedBitmap32 bitmap2 = new EWAHCompressedBitmap32(); - bitmap2.set(1); - bitmap2.set(11); - bitmap2.set(111); - bitmap2.set(1111111); - bitmap2.set(11111111); - Assert.assertTrue(bitmap.intersects(bitmap2)); - Assert.assertTrue(bitmap2.intersects(bitmap)); - - EWAHCompressedBitmap32 bitmap3 = new EWAHCompressedBitmap32(); - bitmap3.set(101); - EWAHCompressedBitmap32 bitmap4 = new EWAHCompressedBitmap32(); - for (int i = 0; i < 100; i++) { - bitmap4.set(i); - } - Assert.assertFalse(bitmap3.intersects(bitmap4)); - Assert.assertFalse(bitmap4.intersects(bitmap3)); - - EWAHCompressedBitmap32 bitmap5 = new EWAHCompressedBitmap32(); - bitmap5.set(0); - bitmap5.set(10); - bitmap5.set(20); - EWAHCompressedBitmap32 bitmap6 = new EWAHCompressedBitmap32(); - bitmap6.set(1); - bitmap6.set(11); - bitmap6.set(21); - bitmap6.set(1111111); - bitmap6.set(11111111); - Assert.assertFalse(bitmap5.intersects(bitmap6)); - Assert.assertFalse(bitmap6.intersects(bitmap5)); - - bitmap5.set(21); - Assert.assertTrue(bitmap5.intersects(bitmap6)); - Assert.assertTrue(bitmap6.intersects(bitmap5)); - - EWAHCompressedBitmap32 bitmap7 = new EWAHCompressedBitmap32(); - bitmap7.set(1); - bitmap7.set(10); - bitmap7.set(20); - bitmap7.set(1111111); - bitmap7.set(11111111); - EWAHCompressedBitmap32 bitmap8 = new EWAHCompressedBitmap32(); - for (int i = 0; i < 1000; i++) { - if (i != 1 && i!=10 && i!=20){ - bitmap8.set(i); - } - } - Assert.assertFalse(bitmap7.intersects(bitmap8)); - Assert.assertFalse(bitmap8.intersects(bitmap7)); - } - - /** - * as per renaud.delbru, Feb 12, 2009 this might throw an error out of bound - * exception. - */ - @Test - public void testLargeEWAHCompressedBitmap() { - System.out.println("testing EWAH over a large array"); - EWAHCompressedBitmap32 myarray1 = new EWAHCompressedBitmap32(); - int N = 11000000; - for (int i = 0; i < N; ++i) { - myarray1.set(i); - } - Assert.assertTrue(myarray1.sizeInBits() == N); - } - - - /** - * Test massive and. - */ - @Test - public void testMassiveAnd() { - System.out.println("testing massive logical and"); - EWAHCompressedBitmap32[] ewah = new EWAHCompressedBitmap32[1024]; - for (int k = 0; k < ewah.length; ++k) - ewah[k] = new EWAHCompressedBitmap32(); - for (int k = 0; k < 30000; ++k) { - ewah[(k + 2 * k * k) % ewah.length].set(k); - } - EWAHCompressedBitmap32 answer = ewah[0]; - for (int k = 1; k < ewah.length; ++k) - answer = answer.and(ewah[k]); - // result should be empty - if (answer.getPositions().size() != 0) - System.out.println(answer.toDebugString()); - Assert.assertTrue(answer.getPositions().size() == 0); - Assert - .assertTrue(EWAHCompressedBitmap32.and(ewah).getPositions().size() == 0); - } - - /** - * Test massive and not. - */ - @Test - public void testMassiveAndNot() { - System.out.println("testing massive and not"); - final int N = 1024; - EWAHCompressedBitmap32[] ewah = new EWAHCompressedBitmap32[N]; - for (int k = 0; k < ewah.length; ++k) - ewah[k] = new EWAHCompressedBitmap32(); - for (int k = 0; k < 30000; ++k) { - ewah[(k + 2 * k * k) % ewah.length].set(k); - } - EWAHCompressedBitmap32 answer = ewah[0]; - EWAHCompressedBitmap32 answer2 = ewah[0]; - for (int k = 1; k < ewah.length; ++k) { - answer = answer.andNot(ewah[k]); - EWAHCompressedBitmap32 copy = null; - try { - copy = (EWAHCompressedBitmap32) ewah[k].clone(); - copy.not(); - answer2.and(copy); - assertEqualsPositions(answer, answer2); - } catch (CloneNotSupportedException e) { - e.printStackTrace(); - } - } - } - - - @Test - public void testsetSizeInBits() { - System.out.println("testing setSizeInBits"); - for(int k = 0; k < 4096; ++k) { - EWAHCompressedBitmap32 ewah = new EWAHCompressedBitmap32(); - ewah.setSizeInBits(k); - Assert.assertEquals(ewah.sizeinbits,k); - Assert.assertEquals(ewah.cardinality(),0); - EWAHCompressedBitmap32 ewah2 = new EWAHCompressedBitmap32(); - ewah2.setSizeInBits(k, false); - Assert.assertEquals(ewah2.sizeinbits,k); - Assert.assertEquals(ewah2.cardinality(),0); - EWAHCompressedBitmap32 ewah3 = new EWAHCompressedBitmap32(); - for(int i = 0; i < k ; ++i) { - ewah3.set(i); - } - Assert.assertEquals(ewah3.sizeinbits,k); - Assert.assertEquals(ewah3.cardinality(),k); - EWAHCompressedBitmap32 ewah4 = new EWAHCompressedBitmap32(); - ewah4.setSizeInBits(k, true); - Assert.assertEquals(ewah4.sizeinbits,k); - Assert.assertEquals(ewah4.cardinality(),k); - } - } - - - /** - * Test massive or. - */ - @Test - public void testMassiveOr() { - System.out - .println("testing massive logical or (can take a couple of minutes)"); - final int N = 128; - for (int howmany = 512; howmany <= 10000; howmany *= 2) { - EWAHCompressedBitmap32[] ewah = new EWAHCompressedBitmap32[N]; - BitSet[] bset = new BitSet[N]; - for (int k = 0; k < ewah.length; ++k) - ewah[k] = new EWAHCompressedBitmap32(); - for (int k = 0; k < bset.length; ++k) - bset[k] = new BitSet(); - for (int k = 0; k < N; ++k) - assertEqualsPositions(bset[k], ewah[k]); - for (int k = 0; k < howmany; ++k) { - ewah[(k + 2 * k * k) % ewah.length].set(k); - bset[(k + 2 * k * k) % ewah.length].set(k); - } - for (int k = 0; k < N; ++k) - assertEqualsPositions(bset[k], ewah[k]); - EWAHCompressedBitmap32 answer = ewah[0]; - BitSet bitsetanswer = bset[0]; - for (int k = 1; k < ewah.length; ++k) { - EWAHCompressedBitmap32 tmp = answer.or(ewah[k]); - bitsetanswer.or(bset[k]); - answer = tmp; - assertEqualsPositions(bitsetanswer, answer); - } - assertEqualsPositions(bitsetanswer, answer); - assertEqualsPositions(bitsetanswer, EWAHCompressedBitmap32.or(ewah)); - int k = 0; - for (int j : answer) { - if (k != j) - System.out.println(answer.toDebugString()); - Assert.assertEquals(k, j); - k += 1; - } - } - } - - /** - * Test massive xor. - */ - @Test - public void testMassiveXOR() { - System.out.println("testing massive xor (can take a couple of minutes)"); - final int N = 16; - EWAHCompressedBitmap32[] ewah = new EWAHCompressedBitmap32[N]; - BitSet[] bset = new BitSet[N]; - for (int k = 0; k < ewah.length; ++k) - ewah[k] = new EWAHCompressedBitmap32(); - for (int k = 0; k < bset.length; ++k) - bset[k] = new BitSet(); - for (int k = 0; k < 30000; ++k) { - ewah[(k + 2 * k * k) % ewah.length].set(k); - bset[(k + 2 * k * k) % ewah.length].set(k); - } - EWAHCompressedBitmap32 answer = ewah[0]; - BitSet bitsetanswer = bset[0]; - for (int k = 1; k < ewah.length; ++k) { - answer = answer.xor(ewah[k]); - bitsetanswer.xor(bset[k]); - assertEqualsPositions(bitsetanswer, answer); - } - int k = 0; - for (int j : answer) { - if (k != j) - System.out.println(answer.toDebugString()); - Assert.assertEquals(k, j); - k += 1; - } - } - - @Test - public void testMultiAnd() { - System.out.println("testing MultiAnd"); - // test bitmap3 has a literal word while bitmap1/2 have a run of 1 - EWAHCompressedBitmap32 bitmap1 = new EWAHCompressedBitmap32(); - bitmap1.addStreamOfEmptyWords(true, 1000); - EWAHCompressedBitmap32 bitmap2 = new EWAHCompressedBitmap32(); - bitmap2.addStreamOfEmptyWords(true, 2000); - EWAHCompressedBitmap32 bitmap3 = new EWAHCompressedBitmap32(); - bitmap3.set(500); - bitmap3.set(502); - bitmap3.set(504); - - assertAndEquals(bitmap1, bitmap2, bitmap3); - - // equal - bitmap1 = new EWAHCompressedBitmap32(); - bitmap1.set(35); - bitmap2 = new EWAHCompressedBitmap32(); - bitmap2.set(35); - bitmap3 = new EWAHCompressedBitmap32(); - bitmap3.set(35); - - assertAndEquals(bitmap1, bitmap2, bitmap3); - - // same number of words for each - bitmap3.set(63); - assertAndEquals(bitmap1, bitmap2, bitmap3); - - // one word bigger - bitmap3.set(64); - assertAndEquals(bitmap1, bitmap2, bitmap3); - - // two words bigger - bitmap3.set(130); - assertAndEquals(bitmap1, bitmap2, bitmap3); - - // test that result can still be appended to - EWAHCompressedBitmap32 resultBitmap = EWAHCompressedBitmap32.and(bitmap1, - bitmap2, bitmap3); - resultBitmap.set(131); - - bitmap1.set(131); - assertEquals(bitmap1, resultBitmap); - - final int N = 128; - for (int howmany = 512; howmany <= 10000; howmany *= 2) { - EWAHCompressedBitmap32[] ewah = new EWAHCompressedBitmap32[N]; - for (int k = 0; k < ewah.length; ++k) - ewah[k] = new EWAHCompressedBitmap32(); - for (int k = 0; k < howmany; ++k) { - ewah[(k + 2 * k * k) % ewah.length].set(k); - } - for (int k = 1; k <= ewah.length; ++k) { - EWAHCompressedBitmap32[] shortewah = new EWAHCompressedBitmap32[k]; - for (int i = 0; i < k; ++i) - shortewah[i] = ewah[i]; - assertAndEquals(shortewah); - } - } - } - - @Test - public void testMultiOr() { - System.out.println("testing MultiOr"); - // test bitmap3 has a literal word while bitmap1/2 have a run of 0 - EWAHCompressedBitmap32 bitmap1 = new EWAHCompressedBitmap32(); - bitmap1.set(1000); - EWAHCompressedBitmap32 bitmap2 = new EWAHCompressedBitmap32(); - bitmap2.set(2000); - EWAHCompressedBitmap32 bitmap3 = new EWAHCompressedBitmap32(); - bitmap3.set(500); - bitmap3.set(502); - bitmap3.set(504); - - EWAHCompressedBitmap32 expected = bitmap1.or(bitmap2).or(bitmap3); - - assertEquals(expected, EWAHCompressedBitmap32.or(bitmap1, bitmap2, bitmap3)); - - final int N = 128; - for (int howmany = 512; howmany <= 10000; howmany *= 2) { - EWAHCompressedBitmap32[] ewah = new EWAHCompressedBitmap32[N]; - for (int k = 0; k < ewah.length; ++k) - ewah[k] = new EWAHCompressedBitmap32(); - for (int k = 0; k < howmany; ++k) { - ewah[(k + 2 * k * k) % ewah.length].set(k); - } - for (int k = 1; k <= ewah.length; ++k) { - EWAHCompressedBitmap32[] shortewah = new EWAHCompressedBitmap32[k]; - for (int i = 0; i < k; ++i) - shortewah[i] = ewah[i]; - assertOrEquals(shortewah); - } - } - - } - - /** - * Test not. (Based on an idea by Ciaran Jessup) - */ - @Test - public void testNot() { - System.out.println("testing not"); - EWAHCompressedBitmap32 ewah = new EWAHCompressedBitmap32(); - for (int i = 0; i <= 184; ++i) { - ewah.set(i); - } - Assert.assertEquals(ewah.cardinality(), 185); - ewah.not(); - Assert.assertEquals(ewah.cardinality(), 0); - } - - @Test - public void testOrCardinality() { - System.out.println("testing Or Cardinality"); - for (int N = 0; N < 1024; ++N) { - EWAHCompressedBitmap32 bitmap = new EWAHCompressedBitmap32(); - for (int i = 0; i < N; i++) { - bitmap.set(i); - } - bitmap.set(1025); - bitmap.set(1026); - Assert.assertEquals(N+2, bitmap.cardinality()); - EWAHCompressedBitmap32 orbitmap = bitmap.or(bitmap); - assertEquals(orbitmap,bitmap); - Assert.assertEquals(N+2, orbitmap.cardinality()); - if(N+2 != bitmap.orCardinality(new EWAHCompressedBitmap32())) { - System.out.println("N = "+N); - System.out.println(bitmap.toDebugString()); - System.out.println("cardinality = "+bitmap.cardinality()); - System.out.println("orCardinality = "+bitmap.orCardinality(new EWAHCompressedBitmap32())); - } - - Assert.assertEquals(N+2, - bitmap.orCardinality(new EWAHCompressedBitmap32())); - } - } - - /** - * Test running length word. - */ - @Test - public void testRunningLengthWord() { - System.out.println("testing RunningLengthWord32"); - int x[] = new int[1]; - RunningLengthWord32 rlw = new RunningLengthWord32(x, 0); - Assert.assertEquals(0, rlw.getNumberOfLiteralWords()); - Assert.assertEquals(false, rlw.getRunningBit()); - Assert.assertEquals(0, rlw.getRunningLength()); - rlw.setRunningBit(true); - Assert.assertEquals(0, rlw.getNumberOfLiteralWords()); - Assert.assertEquals(true, rlw.getRunningBit()); - Assert.assertEquals(0, rlw.getRunningLength()); - rlw.setRunningBit(false); - Assert.assertEquals(0, rlw.getNumberOfLiteralWords()); - Assert.assertEquals(false, rlw.getRunningBit()); - Assert.assertEquals(0, rlw.getRunningLength()); - for (int rl = RunningLengthWord32.largestliteralcount; rl >= 0; rl -= 1024) { - rlw.setNumberOfLiteralWords(rl); - Assert.assertEquals(rl, rlw.getNumberOfLiteralWords()); - Assert.assertEquals(false, rlw.getRunningBit()); - Assert.assertEquals(0, rlw.getRunningLength()); - rlw.setNumberOfLiteralWords(0); - Assert.assertEquals(0, rlw.getNumberOfLiteralWords()); - Assert.assertEquals(false, rlw.getRunningBit()); - Assert.assertEquals(0, rlw.getRunningLength()); - } - for (int rl = 0; rl <= RunningLengthWord32.largestrunninglengthcount; rl += 1024) { - rlw.setRunningLength(rl); - Assert.assertEquals(0, rlw.getNumberOfLiteralWords()); - Assert.assertEquals(false, rlw.getRunningBit()); - Assert.assertEquals(rl, rlw.getRunningLength()); - rlw.setRunningLength(0); - Assert.assertEquals(0, rlw.getNumberOfLiteralWords()); - Assert.assertEquals(false, rlw.getRunningBit()); - Assert.assertEquals(0, rlw.getRunningLength()); - } - rlw.setRunningBit(true); - for (int rl = 0; rl <= RunningLengthWord32.largestrunninglengthcount; rl += 1024) { - rlw.setRunningLength(rl); - Assert.assertEquals(0, rlw.getNumberOfLiteralWords()); - Assert.assertEquals(true, rlw.getRunningBit()); - Assert.assertEquals(rl, rlw.getRunningLength()); - rlw.setRunningLength(0); - Assert.assertEquals(0, rlw.getNumberOfLiteralWords()); - Assert.assertEquals(true, rlw.getRunningBit()); - Assert.assertEquals(0, rlw.getRunningLength()); - } - for (int rl = 0; rl <= RunningLengthWord32.largestliteralcount; rl += 128) { - rlw.setNumberOfLiteralWords(rl); - Assert.assertEquals(rl, rlw.getNumberOfLiteralWords()); - Assert.assertEquals(true, rlw.getRunningBit()); - Assert.assertEquals(0, rlw.getRunningLength()); - rlw.setNumberOfLiteralWords(0); - Assert.assertEquals(0, rlw.getNumberOfLiteralWords()); - Assert.assertEquals(true, rlw.getRunningBit()); - Assert.assertEquals(0, rlw.getRunningLength()); - } - } - - /** - * Test sets and gets. - */ - @Test - public void testSetGet() { - System.out.println("testing EWAH set/get"); - EWAHCompressedBitmap32 ewcb = new EWAHCompressedBitmap32(); - int[] val = { 5, 4400, 44600, 55400, 1000000 }; - for (int k = 0; k < val.length; ++k) { - ewcb.set(val[k]); - } - List result = ewcb.getPositions(); - Assert.assertTrue(val.length == result.size()); - for (int k = 0; k < val.length; ++k) { - Assert.assertEquals(result.get(k).intValue(), val[k]); - } - } - - @Test - public void testHashCode() { - System.out.println("testing hashCode"); - EWAHCompressedBitmap32 ewcb = EWAHCompressedBitmap32.bitmapOf(50, 70).and(EWAHCompressedBitmap32.bitmapOf(50, 1000)); - Assert.assertEquals(EWAHCompressedBitmap32.bitmapOf(50), ewcb); - Assert.assertEquals(EWAHCompressedBitmap32.bitmapOf(50).hashCode(), ewcb.hashCode()); - } - - @Test - public void testSetSizeInBits() { - System.out.println("testing SetSizeInBits"); - testSetSizeInBits(130, 131); - testSetSizeInBits(63, 64); - testSetSizeInBits(64, 65); - testSetSizeInBits(64, 128); - testSetSizeInBits(35, 131); - testSetSizeInBits(130, 400); - testSetSizeInBits(130, 191); - testSetSizeInBits(130, 192); - EWAHCompressedBitmap32 bitmap = new EWAHCompressedBitmap32(); - bitmap.set(31); - bitmap.setSizeInBits(130, false); - bitmap.set(131); - BitSet jdkBitmap = new BitSet(); - jdkBitmap.set(31); - jdkBitmap.set(131); - assertEquals(jdkBitmap, bitmap); - } - - /** - * Test with parameters. - * - * @throws IOException - * Signals that an I/O exception has occurred. - */ - @Test - public void testWithParameters() throws IOException { - System.out - .println("These tests can run for several minutes. Please be patient."); - for (int k = 2; k < 1 << 24; k *= 8) - shouldSetBits(k); - PolizziTest(64); - PolizziTest(128); - PolizziTest(256); - PolizziTest(2048); - System.out.println("Your code is probably ok."); - } - - /** - * Pseudo-non-deterministic test inspired by S.J.vanSchaik. (Yes, - * non-deterministic tests are bad, but the test is actually deterministic.) - */ - @Test - public void vanSchaikTest() { - System.out.println("testing vanSchaikTest (this takes some time)"); - final int totalNumBits = 32768; - final double odds = 0.9; - Random rand = new Random(323232323); - for (int t = 0; t < 100; t++) { - int numBitsSet = 0; - EWAHCompressedBitmap32 cBitMap = new EWAHCompressedBitmap32(); - for (int i = 0; i < totalNumBits; i++) { - if (rand.nextDouble() < odds) { - cBitMap.set(i); - numBitsSet++; + + @Test + public void testGet() { + for (int gap = 29; gap < 10000; gap *= 10) { + EWAHCompressedBitmap32 x = new EWAHCompressedBitmap32(); + for (int k = 0; k < 100; ++k) + x.set(k * gap); + for (int k = 0; k < 100 * gap; ++k) + if (x.get(k)) { + if (k % gap != 0) + throw new RuntimeException( + "spotted an extra set bit at " + + k + " gap = " + + gap); + } else if (k % gap == 0) + throw new RuntimeException( + "missed a set bit " + k + + " gap = " + gap); + } } - } - Assert.assertEquals(cBitMap.cardinality(), numBitsSet); - } - - } - - /** - * Function used in a test inspired by Federico Fissore. - * - * @param size - * the number of set bits - * @param seed - * the random seed - * @return the pseudo-random array int[] - */ - public static int[] createSortedIntArrayOfBitsToSet(int size, int seed) { - Random random = new Random(seed); - // build raw int array - int[] bits = new int[size]; - for (int i = 0; i < bits.length; i++) { - bits[i] = random.nextInt(TEST_BS_SIZE); - } - // might generate duplicates - Arrays.sort(bits); - // first count how many distinct values - int counter = 0; - int oldx = -1; - for (int x : bits) { - if (x != oldx) - ++counter; - oldx = x; - } - // then construct new array - int[] answer = new int[counter]; - counter = 0; - oldx = -1; - for (int x : bits) { - if (x != oldx) { - answer[counter] = x; - ++counter; - } - oldx = x; - } - return answer; - } - - - /** - * Test inspired by Bilal Tayara - */ - @Test - public void TayaraTest() { - System.out.println("Tayara test"); - for(int offset = 64; offset<(1<<30);offset*=2){ - EWAHCompressedBitmap32 a = new EWAHCompressedBitmap32(); - EWAHCompressedBitmap32 b = new EWAHCompressedBitmap32(); - for(int k = 0; k< 64; ++k) { - a.set(offset+k); - b.set(offset+k); - } - if(!a.and(b).equals(a)) throw new RuntimeException("bug"); - if(!a.or(b).equals(a)) throw new RuntimeException("bug"); - } - } + + @SuppressWarnings({ "deprecation", "boxing" }) + @Test + public void OKaserBugReportJuly2013() { + System.out.println("testing OKaserBugReportJuly2013"); + int[][] data = { {}, { 5, 6, 7, 8, 9 }, { 1 }, { 2 }, { 2, 5, 7 }, + { 1 }, { 2 }, { 1, 6, 9 }, { 1, 3, 4, 6, 8, 9 }, + { 1, 3, 4, 6, 8, 9 }, { 1, 3, 6, 8, 9 }, { 2, 5, 7 }, + { 2, 5, 7 }, { 1, 3, 9 }, { 3, 8, 9 } }; + + EWAHCompressedBitmap32[] toBeOred = new EWAHCompressedBitmap32[data.length]; + Set bruteForceAnswer = new HashSet(); + for (int i = 0; i < toBeOred.length; ++i) { + toBeOred[i] = new EWAHCompressedBitmap32(); + for (int j : data[i]) { + toBeOred[i].set(j); + bruteForceAnswer.add(j); + } + toBeOred[i].setSizeInBits(1000,false); + } + + long rightcard = bruteForceAnswer.size(); + EWAHCompressedBitmap32 foo = new EWAHCompressedBitmap32(); + FastAggregation32.legacy_orWithContainer(foo, toBeOred); + Assert.assertEquals(rightcard, foo.cardinality()); + EWAHCompressedBitmap32 e1 = FastAggregation.or(toBeOred); + Assert.assertEquals(rightcard, e1.cardinality()); + EWAHCompressedBitmap32 e2 = FastAggregation32.bufferedor(65536, + toBeOred); + Assert.assertEquals(rightcard, e2.cardinality()); + } + + @Test + public void testSizeInBitsWithAnd() { + System.out.println("testing SizeInBitsWithAnd"); + EWAHCompressedBitmap32 a = new EWAHCompressedBitmap32(); + EWAHCompressedBitmap32 b = new EWAHCompressedBitmap32(); + + a.set(1); + a.set(2); + a.set(3); + + b.set(3); + b.set(4); + b.set(5); + + a.setSizeInBits(10); + b.setSizeInBits(10); + + EWAHCompressedBitmap32 and = a.and(b); + Assert.assertEquals(10, and.sizeInBits()); + EWAHCompressedBitmap32 and2 = EWAHCompressedBitmap32.and(a,b); + Assert.assertEquals(10, and2.sizeInBits()); + } + @Test + public void testSizeInBitsWithAndNot() { + System.out.println("testing SizeInBitsWithAndNot"); + EWAHCompressedBitmap32 a = new EWAHCompressedBitmap32(); + EWAHCompressedBitmap32 b = new EWAHCompressedBitmap32(); + + a.set(1); + a.set(2); + a.set(3); + + b.set(3); + b.set(4); + b.set(5); + + a.setSizeInBits(10); + b.setSizeInBits(10); + + EWAHCompressedBitmap32 and = a.andNot(b); + Assert.assertEquals(10, and.sizeInBits()); + } + + @Test + public void testSizeInBitsWithOr() { + System.out.println("testing SizeInBitsWithOr"); + EWAHCompressedBitmap32 a = new EWAHCompressedBitmap32(); + EWAHCompressedBitmap32 b = new EWAHCompressedBitmap32(); + + a.set(1); + a.set(2); + a.set(3); + + b.set(3); + b.set(4); + b.set(5); + + a.setSizeInBits(10); + b.setSizeInBits(10); + + EWAHCompressedBitmap32 or = a.or(b); + Assert.assertEquals(10, or.sizeInBits()); + EWAHCompressedBitmap32 or2 = EWAHCompressedBitmap32.or(a,b); + Assert.assertEquals(10, or2.sizeInBits()); + } + + + @Test + public void testSizeInBitsWithXor() { + System.out.println("testing SizeInBitsWithXor"); + EWAHCompressedBitmap32 a = new EWAHCompressedBitmap32(); + EWAHCompressedBitmap32 b = new EWAHCompressedBitmap32(); + + a.set(1); + a.set(2); + a.set(3); + + b.set(3); + b.set(4); + b.set(5); + + a.setSizeInBits(10); + b.setSizeInBits(10); + + EWAHCompressedBitmap32 xor = a.xor(b); + Assert.assertEquals(10, xor.sizeInBits()); + EWAHCompressedBitmap32 xor2 = EWAHCompressedBitmap32.xor(a,b); + Assert.assertEquals(10, xor2.sizeInBits()); + } + + + @Test + public void testDebugSetSizeInBitsTest() { + System.out.println("testing DebugSetSizeInBits"); + EWAHCompressedBitmap32 b = new EWAHCompressedBitmap32(); + + b.set(4); + + b.setSizeInBits(6, true); + + List positions = b.getPositions(); + + Assert.assertEquals(2, positions.size()); + Assert.assertEquals(Integer.valueOf(4), positions.get(0)); + Assert.assertEquals(Integer.valueOf(5), positions.get(1)); + + Iterator iterator = b.iterator(); + Assert.assertTrue(iterator.hasNext()); + Assert.assertEquals(Integer.valueOf(4), iterator.next()); + Assert.assertTrue(iterator.hasNext()); + Assert.assertEquals(Integer.valueOf(5), iterator.next()); + Assert.assertFalse(iterator.hasNext()); + + IntIterator intIterator = b.intIterator(); + Assert.assertTrue(intIterator.hasNext()); + Assert.assertEquals(4, intIterator.next()); + Assert.assertTrue(intIterator.hasNext()); + Assert.assertEquals(5, intIterator.next()); + Assert.assertFalse(intIterator.hasNext()); + + } + + /** + * Created: 2/4/11 6:03 PM By: Arnon Moscona. + */ + @Test + public void EwahIteratorProblem() { + System.out.println("testing ArnonMoscona"); + EWAHCompressedBitmap32 bitmap = new EWAHCompressedBitmap32(); + for (int i = 9434560; i <= 9435159; i++) { + bitmap.set(i); + } + IntIterator iterator = bitmap.intIterator(); + List v = bitmap.getPositions(); + int[] array = bitmap.toArray(); + for (int k = 0; k < v.size(); ++k) { + Assert.assertTrue(array[k] == v.get(k).intValue()); + Assert.assertTrue(iterator.hasNext()); + final int ival = iterator.next(); + final int vval = v.get(k).intValue(); + Assert.assertTrue(ival == vval); + } + Assert.assertTrue(!iterator.hasNext()); + // + for (int k = 2; k <= 1024; k *= 2) { + int[] bitsToSet = createSortedIntArrayOfBitsToSet(k, 434455 + 5 * k); + EWAHCompressedBitmap32 ewah = new EWAHCompressedBitmap32(); + for (int i : bitsToSet) { + ewah.set(i); + } + equal(ewah.iterator(), bitsToSet); + } + } + + /** + * Test submitted by Gregory Ssi-Yan-Kai + */ + @Test + public void SsiYanKaiTest() { + System.out.println("testing SsiYanKaiTest"); + EWAHCompressedBitmap32 a = EWAHCompressedBitmap32.bitmapOf(39935, + 39936, 39937, 39938, 39939, 39940, 39941, 39942, 39943, 39944, + 39945, 39946, 39947, 39948, 39949, 39950, 39951, 39952, 39953, + 39954, 39955, 39956, 39957, 39958, 39959, 39960, 39961, 39962, + 39963, 39964, 39965, 39966, 39967, 39968, 39969, 39970, 39971, + 39972, 39973, 39974, 39975, 39976, 39977, 39978, 39979, 39980, + 39981, 39982, 39983, 39984, 39985, 39986, 39987, 39988, 39989, + 39990, 39991, 39992, 39993, 39994, 39995, 39996, 39997, 39998, + 39999, 40000, 40001, 40002, 40003, 40004, 40005, 40006, 40007, + 40008, 40009, 40010, 40011, 40012, 40013, 40014, 40015, 40016, + 40017, 40018, 40019, 40020, 40021, 40022, 40023, 40024, 40025, + 40026, 40027, 40028, 40029, 40030, 40031, 40032, 40033, 40034, + 40035, 40036, 40037, 40038, 40039, 40040, 40041, 40042, 40043, + 40044, 40045, 40046, 40047, 40048, 40049, 40050, 40051, 40052, + 40053, 40054, 40055, 40056, 40057, 40058, 40059, 40060, 40061, + 40062, 40063, 40064, 40065, 40066, 40067, 40068, 40069, 40070, + 40071, 40072, 40073, 40074, 40075, 40076, 40077, 40078, 40079, + 40080, 40081, 40082, 40083, 40084, 40085, 40086, 40087, 40088, + 40089, 40090, 40091, 40092, 40093, 40094, 40095, 40096, 40097, + 40098, 40099, 40100); + EWAHCompressedBitmap32 b = EWAHCompressedBitmap32.bitmapOf(39935, + 39936, 39937, 39938, 39939, 39940, 39941, 39942, 39943, 39944, + 39945, 39946, 39947, 39948, 39949, 39950, 39951, 39952, 39953, + 39954, 39955, 39956, 39957, 39958, 39959, 39960, 39961, 39962, + 39963, 39964, 39965, 39966, 39967, 39968, 39969, 39970, 39971, + 39972, 39973, 39974, 39975, 39976, 39977, 39978, 39979, 39980, + 39981, 39982, 39983, 39984, 39985, 39986, 39987, 39988, 39989, + 39990, 39991, 39992, 39993, 39994, 39995, 39996, 39997, 39998, + 39999, 270000); + LinkedHashSet aPositions = new LinkedHashSet( + a.getPositions()); + int intersection = 0; + EWAHCompressedBitmap32 inter = new EWAHCompressedBitmap32(); + LinkedHashSet bPositions = new LinkedHashSet( + b.getPositions()); + for (Integer integer : bPositions) { + if (aPositions.contains(integer)) { + inter.set(integer.intValue()); + ++intersection; + } + } + EWAHCompressedBitmap32 and2 = a.and(b); + if (!and2.equals(inter)) + throw new RuntimeException("intersections don't match"); + if (intersection != and2.cardinality()) + throw new RuntimeException("cardinalities don't match"); + } + + /** + * Test inspired by William Habermaas. + */ + @Test + public void habermaasTest() { + System.out.println("testing habermaasTest"); + BitSet bitsetaa = new BitSet(); + EWAHCompressedBitmap32 aa = new EWAHCompressedBitmap32(); + int[] val = { 55400, 1000000, 1000128 }; + for (int k = 0; k < val.length; ++k) { + aa.set(val[k]); + bitsetaa.set(val[k]); + } + equal(aa, bitsetaa); + BitSet bitsetab = new BitSet(); + EWAHCompressedBitmap32 ab = new EWAHCompressedBitmap32(); + for (int i = 4096; i < (4096 + 5); i++) { + ab.set(i); + bitsetab.set(i); + } + ab.set(99000); + bitsetab.set(99000); + ab.set(1000130); + bitsetab.set(1000130); + equal(ab, bitsetab); + EWAHCompressedBitmap32 bb = aa.or(ab); + EWAHCompressedBitmap32 bbAnd = aa.and(ab); + try { + EWAHCompressedBitmap32 abnot = ab.clone(); + abnot.not(); + EWAHCompressedBitmap32 bbAnd2 = aa.andNot(abnot); + assertEquals(bbAnd2, bbAnd); + } catch (CloneNotSupportedException e) { + e.printStackTrace(); + } + BitSet bitsetbb = (BitSet) bitsetaa.clone(); + bitsetbb.or(bitsetab); + BitSet bitsetbbAnd = (BitSet) bitsetaa.clone(); + bitsetbbAnd.and(bitsetab); + equal(bbAnd, bitsetbbAnd); + equal(bb, bitsetbb); + } + + @Test + public void testAndResultAppend() { + System.out.println("testing AndResultAppend"); + EWAHCompressedBitmap32 bitmap1 = new EWAHCompressedBitmap32(); + bitmap1.set(35); + EWAHCompressedBitmap32 bitmap2 = new EWAHCompressedBitmap32(); + bitmap2.set(35); + bitmap2.set(130); + + EWAHCompressedBitmap32 resultBitmap = bitmap1.and(bitmap2); + resultBitmap.set(131); + + bitmap1.set(131); + assertEquals(bitmap1, resultBitmap); + } + + /** + * Test cardinality. + */ + @Test + public void testCardinality() { + System.out.println("testing EWAH cardinality"); + EWAHCompressedBitmap32 bitmap = new EWAHCompressedBitmap32(); + bitmap.set(Integer.MAX_VALUE - 32); + // System.out.format("Total Items %d\n", bitmap.cardinality()); + Assert.assertTrue(bitmap.cardinality() == 1); + } + + /** + * Test clear function + */ + @Test + public void testClear() { + System.out.println("testing Clear"); + EWAHCompressedBitmap32 bitmap = new EWAHCompressedBitmap32(); + bitmap.set(5); + bitmap.clear(); + bitmap.set(7); + Assert.assertTrue(1 == bitmap.cardinality()); + Assert.assertTrue(1 == bitmap.getPositions().size()); + Assert.assertTrue(1 == bitmap.toArray().length); + Assert.assertTrue(7 == bitmap.getPositions().get(0).intValue()); + Assert.assertTrue(7 == bitmap.toArray()[0]); + bitmap.clear(); + bitmap.set(5000); + Assert.assertTrue(1 == bitmap.cardinality()); + Assert.assertTrue(1 == bitmap.getPositions().size()); + Assert.assertTrue(1 == bitmap.toArray().length); + Assert.assertTrue(5000 == bitmap.getPositions().get(0).intValue()); + bitmap.set(5001); + bitmap.set(5005); + bitmap.set(5100); + bitmap.set(5500); + bitmap.clear(); + bitmap.set(5); + bitmap.set(7); + bitmap.set(1000); + bitmap.set(1001); + Assert.assertTrue(4 == bitmap.cardinality()); + List positions = bitmap.getPositions(); + Assert.assertTrue(4 == positions.size()); + Assert.assertTrue(5 == positions.get(0).intValue()); + Assert.assertTrue(7 == positions.get(1).intValue()); + Assert.assertTrue(1000 == positions.get(2).intValue()); + Assert.assertTrue(1001 == positions.get(3).intValue()); + } + + /** + * Test ewah compressed bitmap. + */ + @Test + public void testEWAHCompressedBitmap() { + System.out.println("testing EWAH"); + int zero = 0; + int specialval = 1 | (1 << 4) | (1 << 31); + int notzero = ~zero; + EWAHCompressedBitmap32 myarray1 = new EWAHCompressedBitmap32(); + myarray1.add(zero); + myarray1.add(zero); + myarray1.add(zero); + myarray1.add(specialval); + myarray1.add(specialval); + myarray1.add(notzero); + myarray1.add(zero); + Assert.assertEquals(myarray1.getPositions().size(), 6 + 32); + EWAHCompressedBitmap32 myarray2 = new EWAHCompressedBitmap32(); + myarray2.add(zero); + myarray2.add(specialval); + myarray2.add(specialval); + myarray2.add(notzero); + myarray2.add(zero); + myarray2.add(zero); + myarray2.add(zero); + Assert.assertEquals(myarray2.getPositions().size(), 6 + 32); + List data1 = myarray1.getPositions(); + List data2 = myarray2.getPositions(); + Vector logicalor = new Vector(); + { + HashSet tmp = new HashSet(); + tmp.addAll(data1); + tmp.addAll(data2); + logicalor.addAll(tmp); + } + Collections.sort(logicalor); + Vector logicaland = new Vector(); + logicaland.addAll(data1); + logicaland.retainAll(data2); + Collections.sort(logicaland); + EWAHCompressedBitmap32 arrayand = myarray1.and(myarray2); + Assert.assertTrue(arrayand.getPositions().equals(logicaland)); + EWAHCompressedBitmap32 arrayor = myarray1.or(myarray2); + Assert.assertTrue(arrayor.getPositions().equals(logicalor)); + EWAHCompressedBitmap32 arrayandbis = myarray2.and(myarray1); + Assert.assertTrue(arrayandbis.getPositions().equals(logicaland)); + EWAHCompressedBitmap32 arrayorbis = myarray2.or(myarray1); + Assert.assertTrue(arrayorbis.getPositions().equals(logicalor)); + EWAHCompressedBitmap32 x = new EWAHCompressedBitmap32(); + for (Integer i : myarray1.getPositions()) { + x.set(i.intValue()); + } + Assert.assertTrue(x.getPositions().equals(myarray1.getPositions())); + x = new EWAHCompressedBitmap32(); + for (Integer i : myarray2.getPositions()) { + x.set(i.intValue()); + } + Assert.assertTrue(x.getPositions().equals(myarray2.getPositions())); + x = new EWAHCompressedBitmap32(); + for (Iterator k = myarray1.iterator(); k.hasNext();) { + x.set(extracted(k).intValue()); + } + Assert.assertTrue(x.getPositions().equals(myarray1.getPositions())); + x = new EWAHCompressedBitmap32(); + for (Iterator k = myarray2.iterator(); k.hasNext();) { + x.set(extracted(k).intValue()); + } + Assert.assertTrue(x.getPositions().equals(myarray2.getPositions())); + } + + /** + * Test externalization. + * + * @throws IOException + * Signals that an I/O exception has occurred. + */ + @Test + public void testExternalization() throws IOException { + System.out.println("testing EWAH externalization"); + EWAHCompressedBitmap32 ewcb = new EWAHCompressedBitmap32(); + int[] val = { 5, 4400, 44600, 55400, 1000000 }; + for (int k = 0; k < val.length; ++k) { + ewcb.set(val[k]); + } + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + ObjectOutputStream oo = new ObjectOutputStream(bos); + ewcb.writeExternal(oo); + oo.close(); + ewcb = null; + ewcb = new EWAHCompressedBitmap32(); + ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); + ewcb.readExternal(new ObjectInputStream(bis)); + List result = ewcb.getPositions(); + Assert.assertTrue(val.length == result.size()); + for (int k = 0; k < val.length; ++k) { + Assert.assertTrue(result.get(k).intValue() == val[k]); + } + } + + @Test + public void testExtremeRange() { + System.out.println("testing EWAH at its extreme range"); + EWAHCompressedBitmap32 myarray1 = new EWAHCompressedBitmap32(); + int N = 1024; + for (int i = 0; i < N; ++i) { + myarray1.set(Integer.MAX_VALUE - 32 - N + i); + Assert.assertTrue(myarray1.cardinality() == i + 1); + int[] val = myarray1.toArray(); + Assert.assertTrue(val[0] == Integer.MAX_VALUE - 32 - N); + } + } + + /** + * Test the intersects method + */ + @Test + public void testIntersectsMethod() { + System.out.println("testing Intersets Bug"); + EWAHCompressedBitmap32 bitmap = new EWAHCompressedBitmap32(); + bitmap.set(1); + EWAHCompressedBitmap32 bitmap2 = new EWAHCompressedBitmap32(); + bitmap2.set(1); + bitmap2.set(11); + bitmap2.set(111); + bitmap2.set(1111111); + bitmap2.set(11111111); + Assert.assertTrue(bitmap.intersects(bitmap2)); + Assert.assertTrue(bitmap2.intersects(bitmap)); + + EWAHCompressedBitmap32 bitmap3 = new EWAHCompressedBitmap32(); + bitmap3.set(101); + EWAHCompressedBitmap32 bitmap4 = new EWAHCompressedBitmap32(); + for (int i = 0; i < 100; i++) { + bitmap4.set(i); + } + Assert.assertFalse(bitmap3.intersects(bitmap4)); + Assert.assertFalse(bitmap4.intersects(bitmap3)); + + EWAHCompressedBitmap32 bitmap5 = new EWAHCompressedBitmap32(); + bitmap5.set(0); + bitmap5.set(10); + bitmap5.set(20); + EWAHCompressedBitmap32 bitmap6 = new EWAHCompressedBitmap32(); + bitmap6.set(1); + bitmap6.set(11); + bitmap6.set(21); + bitmap6.set(1111111); + bitmap6.set(11111111); + Assert.assertFalse(bitmap5.intersects(bitmap6)); + Assert.assertFalse(bitmap6.intersects(bitmap5)); + + bitmap5.set(21); + Assert.assertTrue(bitmap5.intersects(bitmap6)); + Assert.assertTrue(bitmap6.intersects(bitmap5)); + + EWAHCompressedBitmap32 bitmap7 = new EWAHCompressedBitmap32(); + bitmap7.set(1); + bitmap7.set(10); + bitmap7.set(20); + bitmap7.set(1111111); + bitmap7.set(11111111); + EWAHCompressedBitmap32 bitmap8 = new EWAHCompressedBitmap32(); + for (int i = 0; i < 1000; i++) { + if (i != 1 && i != 10 && i != 20) { + bitmap8.set(i); + } + } + Assert.assertFalse(bitmap7.intersects(bitmap8)); + Assert.assertFalse(bitmap8.intersects(bitmap7)); + } + + /** + * as per renaud.delbru, Feb 12, 2009 this might throw an error out of bound + * exception. + */ + @Test + public void testLargeEWAHCompressedBitmap() { + System.out.println("testing EWAH over a large array"); + EWAHCompressedBitmap32 myarray1 = new EWAHCompressedBitmap32(); + int N = 11000000; + for (int i = 0; i < N; ++i) { + myarray1.set(i); + } + Assert.assertTrue(myarray1.sizeInBits() == N); + } + + /** + * Test massive and. + */ + @Test + public void testMassiveAnd() { + System.out.println("testing massive logical and"); + EWAHCompressedBitmap32[] ewah = new EWAHCompressedBitmap32[1024]; + for (int k = 0; k < ewah.length; ++k) + ewah[k] = new EWAHCompressedBitmap32(); + for (int k = 0; k < 30000; ++k) { + ewah[(k + 2 * k * k) % ewah.length].set(k); + } + EWAHCompressedBitmap32 answer = ewah[0]; + for (int k = 1; k < ewah.length; ++k) + answer = answer.and(ewah[k]); + // result should be empty + if (answer.getPositions().size() != 0) + System.out.println(answer.toDebugString()); + Assert.assertTrue(answer.getPositions().size() == 0); + Assert.assertTrue(EWAHCompressedBitmap32.and(ewah).getPositions() + .size() == 0); + } + + /** + * Test massive and not. + */ + @Test + public void testMassiveAndNot() { + System.out.println("testing massive and not"); + final int N = 1024; + EWAHCompressedBitmap32[] ewah = new EWAHCompressedBitmap32[N]; + for (int k = 0; k < ewah.length; ++k) + ewah[k] = new EWAHCompressedBitmap32(); + for (int k = 0; k < 30000; ++k) { + ewah[(k + 2 * k * k) % ewah.length].set(k); + } + EWAHCompressedBitmap32 answer = ewah[0]; + EWAHCompressedBitmap32 answer2 = ewah[0]; + for (int k = 1; k < ewah.length; ++k) { + answer = answer.andNot(ewah[k]); + EWAHCompressedBitmap32 copy = null; + try { + copy = ewah[k].clone(); + copy.not(); + answer2.and(copy); + assertEqualsPositions(answer, answer2); + } catch (CloneNotSupportedException e) { + e.printStackTrace(); + } + } + } + + @Test + public void testsetSizeInBits() { + System.out.println("testing setSizeInBits"); + for (int k = 0; k < 4096; ++k) { + EWAHCompressedBitmap32 ewah = new EWAHCompressedBitmap32(); + ewah.setSizeInBits(k,false); + Assert.assertEquals(ewah.sizeinbits, k); + Assert.assertEquals(ewah.cardinality(), 0); + EWAHCompressedBitmap32 ewah2 = new EWAHCompressedBitmap32(); + ewah2.setSizeInBits(k, false); + Assert.assertEquals(ewah2.sizeinbits, k); + Assert.assertEquals(ewah2.cardinality(), 0); + EWAHCompressedBitmap32 ewah3 = new EWAHCompressedBitmap32(); + for (int i = 0; i < k; ++i) { + ewah3.set(i); + } + Assert.assertEquals(ewah3.sizeinbits, k); + Assert.assertEquals(ewah3.cardinality(), k); + EWAHCompressedBitmap32 ewah4 = new EWAHCompressedBitmap32(); + ewah4.setSizeInBits(k, true); + Assert.assertEquals(ewah4.sizeinbits, k); + Assert.assertEquals(ewah4.cardinality(), k); + } + } + + /** + * Test massive or. + */ + @Test + public void testMassiveOr() { + System.out + .println("testing massive logical or (can take a couple of minutes)"); + final int N = 128; + for (int howmany = 512; howmany <= 10000; howmany *= 2) { + EWAHCompressedBitmap32[] ewah = new EWAHCompressedBitmap32[N]; + BitSet[] bset = new BitSet[N]; + for (int k = 0; k < ewah.length; ++k) + ewah[k] = new EWAHCompressedBitmap32(); + for (int k = 0; k < bset.length; ++k) + bset[k] = new BitSet(); + for (int k = 0; k < N; ++k) + assertEqualsPositions(bset[k], ewah[k]); + for (int k = 0; k < howmany; ++k) { + ewah[(k + 2 * k * k) % ewah.length].set(k); + bset[(k + 2 * k * k) % ewah.length].set(k); + } + for (int k = 0; k < N; ++k) + assertEqualsPositions(bset[k], ewah[k]); + EWAHCompressedBitmap32 answer = ewah[0]; + BitSet bitsetanswer = bset[0]; + for (int k = 1; k < ewah.length; ++k) { + EWAHCompressedBitmap32 tmp = answer.or(ewah[k]); + bitsetanswer.or(bset[k]); + answer = tmp; + assertEqualsPositions(bitsetanswer, answer); + } + assertEqualsPositions(bitsetanswer, answer); + assertEqualsPositions(bitsetanswer, EWAHCompressedBitmap32.or(ewah)); + int k = 0; + for (int j : answer) { + if (k != j) + System.out.println(answer.toDebugString()); + Assert.assertEquals(k, j); + k += 1; + } + } + } + + /** + * Test massive xor. + */ + @Test + public void testMassiveXOR() { + System.out + .println("testing massive xor (can take a couple of minutes)"); + final int N = 16; + EWAHCompressedBitmap32[] ewah = new EWAHCompressedBitmap32[N]; + BitSet[] bset = new BitSet[N]; + for (int k = 0; k < ewah.length; ++k) + ewah[k] = new EWAHCompressedBitmap32(); + for (int k = 0; k < bset.length; ++k) + bset[k] = new BitSet(); + for (int k = 0; k < 30000; ++k) { + ewah[(k + 2 * k * k) % ewah.length].set(k); + bset[(k + 2 * k * k) % ewah.length].set(k); + } + EWAHCompressedBitmap32 answer = ewah[0]; + BitSet bitsetanswer = bset[0]; + for (int k = 1; k < ewah.length; ++k) { + answer = answer.xor(ewah[k]); + bitsetanswer.xor(bset[k]); + assertEqualsPositions(bitsetanswer, answer); + } + int k = 0; + for (int j : answer) { + if (k != j) + System.out.println(answer.toDebugString()); + Assert.assertEquals(k, j); + k += 1; + } + } + + @Test + public void testMultiAnd() { + System.out.println("testing MultiAnd"); + // test bitmap3 has a literal word while bitmap1/2 have a run of 1 + EWAHCompressedBitmap32 bitmap1 = new EWAHCompressedBitmap32(); + bitmap1.addStreamOfEmptyWords(true, 1000); + EWAHCompressedBitmap32 bitmap2 = new EWAHCompressedBitmap32(); + bitmap2.addStreamOfEmptyWords(true, 2000); + EWAHCompressedBitmap32 bitmap3 = new EWAHCompressedBitmap32(); + bitmap3.set(500); + bitmap3.set(502); + bitmap3.set(504); + + assertAndEquals(bitmap1, bitmap2, bitmap3); + + // equal + bitmap1 = new EWAHCompressedBitmap32(); + bitmap1.set(35); + bitmap2 = new EWAHCompressedBitmap32(); + bitmap2.set(35); + bitmap3 = new EWAHCompressedBitmap32(); + bitmap3.set(35); + + assertAndEquals(bitmap1, bitmap2, bitmap3); + + // same number of words for each + bitmap3.set(63); + assertAndEquals(bitmap1, bitmap2, bitmap3); + + // one word bigger + bitmap3.set(64); + assertAndEquals(bitmap1, bitmap2, bitmap3); + + // two words bigger + bitmap3.set(130); + assertAndEquals(bitmap1, bitmap2, bitmap3); + + // test that result can still be appended to + EWAHCompressedBitmap32 resultBitmap = EWAHCompressedBitmap32.and( + bitmap1, bitmap2, bitmap3); + resultBitmap.set(131); + + bitmap1.set(131); + assertEquals(bitmap1, resultBitmap); + + final int N = 128; + for (int howmany = 512; howmany <= 10000; howmany *= 2) { + EWAHCompressedBitmap32[] ewah = new EWAHCompressedBitmap32[N]; + for (int k = 0; k < ewah.length; ++k) + ewah[k] = new EWAHCompressedBitmap32(); + for (int k = 0; k < howmany; ++k) { + ewah[(k + 2 * k * k) % ewah.length].set(k); + } + for (int k = 1; k <= ewah.length; ++k) { + EWAHCompressedBitmap32[] shortewah = new EWAHCompressedBitmap32[k]; + for (int i = 0; i < k; ++i) + shortewah[i] = ewah[i]; + assertAndEquals(shortewah); + } + } + } + + @Test + public void testMultiOr() { + System.out.println("testing MultiOr"); + // test bitmap3 has a literal word while bitmap1/2 have a run of 0 + EWAHCompressedBitmap32 bitmap1 = new EWAHCompressedBitmap32(); + bitmap1.set(1000); + EWAHCompressedBitmap32 bitmap2 = new EWAHCompressedBitmap32(); + bitmap2.set(2000); + EWAHCompressedBitmap32 bitmap3 = new EWAHCompressedBitmap32(); + bitmap3.set(500); + bitmap3.set(502); + bitmap3.set(504); + + EWAHCompressedBitmap32 expected = bitmap1.or(bitmap2).or(bitmap3); + + assertEquals(expected, + EWAHCompressedBitmap32.or(bitmap1, bitmap2, bitmap3)); + + final int N = 128; + for (int howmany = 512; howmany <= 10000; howmany *= 2) { + EWAHCompressedBitmap32[] ewah = new EWAHCompressedBitmap32[N]; + for (int k = 0; k < ewah.length; ++k) + ewah[k] = new EWAHCompressedBitmap32(); + for (int k = 0; k < howmany; ++k) { + ewah[(k + 2 * k * k) % ewah.length].set(k); + } + for (int k = 1; k <= ewah.length; ++k) { + EWAHCompressedBitmap32[] shortewah = new EWAHCompressedBitmap32[k]; + for (int i = 0; i < k; ++i) + shortewah[i] = ewah[i]; + assertOrEquals(shortewah); + } + } + + } + + /** + * Test not. (Based on an idea by Ciaran Jessup) + */ + @Test + public void testNot() { + System.out.println("testing not"); + EWAHCompressedBitmap32 ewah = new EWAHCompressedBitmap32(); + for (int i = 0; i <= 184; ++i) { + ewah.set(i); + } + Assert.assertEquals(ewah.cardinality(), 185); + ewah.not(); + Assert.assertEquals(ewah.cardinality(), 0); + } + + @Test + public void testOrCardinality() { + System.out.println("testing Or Cardinality"); + for (int N = 0; N < 1024; ++N) { + EWAHCompressedBitmap32 bitmap = new EWAHCompressedBitmap32(); + for (int i = 0; i < N; i++) { + bitmap.set(i); + } + bitmap.set(1025); + bitmap.set(1026); + Assert.assertEquals(N + 2, bitmap.cardinality()); + EWAHCompressedBitmap32 orbitmap = bitmap.or(bitmap); + assertEquals(orbitmap, bitmap); + Assert.assertEquals(N + 2, orbitmap.cardinality()); + if (N + 2 != bitmap.orCardinality(new EWAHCompressedBitmap32())) { + System.out.println("N = " + N); + System.out.println(bitmap.toDebugString()); + System.out.println("cardinality = " + bitmap.cardinality()); + System.out.println("orCardinality = " + + bitmap.orCardinality(new EWAHCompressedBitmap32())); + } + + Assert.assertEquals(N + 2, + bitmap.orCardinality(new EWAHCompressedBitmap32())); + } + } + + + /** + * Test sets and gets. + */ + @Test + public void testSetGet() { + System.out.println("testing EWAH set/get"); + EWAHCompressedBitmap32 ewcb = new EWAHCompressedBitmap32(); + int[] val = { 5, 4400, 44600, 55400, 1000000 }; + for (int k = 0; k < val.length; ++k) { + ewcb.set(val[k]); + } + List result = ewcb.getPositions(); + Assert.assertTrue(val.length == result.size()); + for (int k = 0; k < val.length; ++k) { + Assert.assertEquals(result.get(k).intValue(), val[k]); + } + } + + @Test + public void testHashCode() { + System.out.println("testing hashCode"); + EWAHCompressedBitmap32 ewcb = EWAHCompressedBitmap32.bitmapOf(50, 70) + .and(EWAHCompressedBitmap32.bitmapOf(50, 1000)); + Assert.assertEquals(EWAHCompressedBitmap32.bitmapOf(50), ewcb); + Assert.assertEquals(EWAHCompressedBitmap32.bitmapOf(50).hashCode(), + ewcb.hashCode()); + } + + @Test + public void testSetSizeInBits() { + System.out.println("testing SetSizeInBits"); + testSetSizeInBits(130, 131); + testSetSizeInBits(63, 64); + testSetSizeInBits(64, 65); + testSetSizeInBits(64, 128); + testSetSizeInBits(35, 131); + testSetSizeInBits(130, 400); + testSetSizeInBits(130, 191); + testSetSizeInBits(130, 192); + EWAHCompressedBitmap32 bitmap = new EWAHCompressedBitmap32(); + bitmap.set(31); + bitmap.setSizeInBits(130, false); + bitmap.set(131); + BitSet jdkBitmap = new BitSet(); + jdkBitmap.set(31); + jdkBitmap.set(131); + assertEquals(jdkBitmap, bitmap); + } - + /** + * Test with parameters. + * + * @throws IOException + * Signals that an I/O exception has occurred. + */ + @Test + public void testWithParameters() throws IOException { + System.out + .println("These tests can run for several minutes. Please be patient."); + for (int k = 2; k < 1 << 24; k *= 8) + shouldSetBits(k); + PolizziTest(64); + PolizziTest(128); + PolizziTest(256); + PolizziTest(2048); + System.out.println("Your code is probably ok."); + } + + /** + * Pseudo-non-deterministic test inspired by S.J.vanSchaik. (Yes, + * non-deterministic tests are bad, but the test is actually deterministic.) + */ + @Test + public void vanSchaikTest() { + System.out.println("testing vanSchaikTest (this takes some time)"); + final int totalNumBits = 32768; + final double odds = 0.9; + Random rand = new Random(323232323); + for (int t = 0; t < 100; t++) { + int numBitsSet = 0; + EWAHCompressedBitmap32 cBitMap = new EWAHCompressedBitmap32(); + for (int i = 0; i < totalNumBits; i++) { + if (rand.nextDouble() < odds) { + cBitMap.set(i); + numBitsSet++; + } + } + Assert.assertEquals(cBitMap.cardinality(), numBitsSet); + } + + } + + /** + * Function used in a test inspired by Federico Fissore. + * + * @param size + * the number of set bits + * @param seed + * the random seed + * @return the pseudo-random array int[] + */ + public static int[] createSortedIntArrayOfBitsToSet(int size, int seed) { + Random random = new Random(seed); + // build raw int array + int[] bits = new int[size]; + for (int i = 0; i < bits.length; i++) { + bits[i] = random.nextInt(TEST_BS_SIZE); + } + // might generate duplicates + Arrays.sort(bits); + // first count how many distinct values + int counter = 0; + int oldx = -1; + for (int x : bits) { + if (x != oldx) + ++counter; + oldx = x; + } + // then construct new array + int[] answer = new int[counter]; + counter = 0; + oldx = -1; + for (int x : bits) { + if (x != oldx) { + answer[counter] = x; + ++counter; + } + oldx = x; + } + return answer; + } + + /** + * Test inspired by Bilal Tayara + */ + @Test + public void TayaraTest() { + System.out.println("Tayara test"); + for (int offset = 64; offset < (1 << 30); offset *= 2) { + EWAHCompressedBitmap32 a = new EWAHCompressedBitmap32(); + EWAHCompressedBitmap32 b = new EWAHCompressedBitmap32(); + for (int k = 0; k < 64; ++k) { + a.set(offset + k); + b.set(offset + k); + } + if (!a.and(b).equals(a)) + throw new RuntimeException("bug"); + if (!a.or(b).equals(a)) + throw new RuntimeException("bug"); + } + } - @Test + @Test public void TestCloneEwahCompressedBitArray() throws CloneNotSupportedException { System.out.println("testing EWAH clone"); @@ -918,422 +1037,428 @@ EWAHCompressedBitmap32 b; - b = (EWAHCompressedBitmap32) a.clone(); + b = a.clone(); a.setSizeInBits(487123, false); b.setSizeInBits(487123, false); Assert.assertTrue(a.equals(b)); } - - /** - * a non-deterministic test proposed by Marc Polizzi. - * - * @param maxlength - * the maximum uncompressed size of the bitmap - */ - public static void PolizziTest(int maxlength) { - System.out.println("Polizzi test with max length = " + maxlength); - for (int k = 0; k < 10000; ++k) { - final Random rnd = new Random(); - final EWAHCompressedBitmap32 ewahBitmap1 = new EWAHCompressedBitmap32(); - final BitSet jdkBitmap1 = new BitSet(); - final EWAHCompressedBitmap32 ewahBitmap2 = new EWAHCompressedBitmap32(); - final BitSet jdkBitmap2 = new BitSet(); - final EWAHCompressedBitmap32 ewahBitmap3 = new EWAHCompressedBitmap32(); - final BitSet jdkBitmap3 = new BitSet(); - final int len = rnd.nextInt(maxlength); - for (int pos = 0; pos < len; pos++) { // random *** number of bits set *** - if (rnd.nextInt(7) == 0) { // random *** increasing *** values - ewahBitmap1.set(pos); - jdkBitmap1.set(pos); - } - if (rnd.nextInt(11) == 0) { // random *** increasing *** values - ewahBitmap2.set(pos); - jdkBitmap2.set(pos); - } - if (rnd.nextInt(7) == 0) { // random *** increasing *** values - ewahBitmap3.set(pos); - jdkBitmap3.set(pos); - } - } - assertEquals(jdkBitmap1, ewahBitmap1); - assertEquals(jdkBitmap2, ewahBitmap2); - assertEquals(jdkBitmap3, ewahBitmap3); - // XOR - { - final EWAHCompressedBitmap32 xorEwahBitmap = ewahBitmap1 - .xor(ewahBitmap2); - final BitSet xorJdkBitmap = (BitSet) jdkBitmap1.clone(); - xorJdkBitmap.xor(jdkBitmap2); - assertEquals(xorJdkBitmap, xorEwahBitmap); - } - // AND - { - final EWAHCompressedBitmap32 andEwahBitmap = ewahBitmap1 - .and(ewahBitmap2); - final BitSet andJdkBitmap = (BitSet) jdkBitmap1.clone(); - andJdkBitmap.and(jdkBitmap2); - assertEquals(andJdkBitmap, andEwahBitmap); - } - // AND - { - final EWAHCompressedBitmap32 andEwahBitmap = ewahBitmap2 - .and(ewahBitmap1); - final BitSet andJdkBitmap = (BitSet) jdkBitmap1.clone(); - andJdkBitmap.and(jdkBitmap2); - assertEquals(andJdkBitmap, andEwahBitmap); - assertEquals(andJdkBitmap, - EWAHCompressedBitmap32.and(ewahBitmap1, ewahBitmap2)); - } - // MULTI AND - { - final BitSet andJdkBitmap = (BitSet) jdkBitmap1.clone(); - andJdkBitmap.and(jdkBitmap2); - andJdkBitmap.and(jdkBitmap3); - assertEquals(andJdkBitmap, - EWAHCompressedBitmap32.and(ewahBitmap1, ewahBitmap2, ewahBitmap3)); - assertEquals(andJdkBitmap, - EWAHCompressedBitmap32.and(ewahBitmap3, ewahBitmap2, ewahBitmap1)); - Assert.assertEquals(andJdkBitmap.cardinality(), EWAHCompressedBitmap32 - .andCardinality(ewahBitmap1, ewahBitmap2, ewahBitmap3)); - } - // AND NOT - { - final EWAHCompressedBitmap32 andNotEwahBitmap = ewahBitmap1 - .andNot(ewahBitmap2); - final BitSet andNotJdkBitmap = (BitSet) jdkBitmap1.clone(); - andNotJdkBitmap.andNot(jdkBitmap2); - assertEquals(andNotJdkBitmap, andNotEwahBitmap); - } - // AND NOT - { - final EWAHCompressedBitmap32 andNotEwahBitmap = ewahBitmap2 - .andNot(ewahBitmap1); - final BitSet andNotJdkBitmap = (BitSet) jdkBitmap2.clone(); - andNotJdkBitmap.andNot(jdkBitmap1); - assertEquals(andNotJdkBitmap, andNotEwahBitmap); - } - // OR - { - final EWAHCompressedBitmap32 orEwahBitmap = ewahBitmap1.or(ewahBitmap2); - final BitSet orJdkBitmap = (BitSet) jdkBitmap1.clone(); - orJdkBitmap.or(jdkBitmap2); - assertEquals(orJdkBitmap, orEwahBitmap); - assertEquals(orJdkBitmap, - EWAHCompressedBitmap32.or(ewahBitmap1, ewahBitmap2)); - Assert.assertEquals(orEwahBitmap.cardinality(), - ewahBitmap1.orCardinality(ewahBitmap2)); - } - // OR - { - final EWAHCompressedBitmap32 orEwahBitmap = ewahBitmap2.or(ewahBitmap1); - final BitSet orJdkBitmap = (BitSet) jdkBitmap1.clone(); - orJdkBitmap.or(jdkBitmap2); - assertEquals(orJdkBitmap, orEwahBitmap); - } - // MULTI OR - { - final BitSet orJdkBitmap = (BitSet) jdkBitmap1.clone(); - orJdkBitmap.or(jdkBitmap2); - orJdkBitmap.or(jdkBitmap3); - assertEquals(orJdkBitmap, - EWAHCompressedBitmap32.or(ewahBitmap1, ewahBitmap2, ewahBitmap3)); - assertEquals(orJdkBitmap, - EWAHCompressedBitmap32.or(ewahBitmap3, ewahBitmap2, ewahBitmap1)); - Assert.assertEquals(orJdkBitmap.cardinality(), EWAHCompressedBitmap32 - .orCardinality(ewahBitmap1, ewahBitmap2, ewahBitmap3)); - } - } - } - - /** - * Pseudo-non-deterministic test inspired by Federico Fissore. - * - * @param length - * the number of set bits in a bitmap - */ - public static void shouldSetBits(int length) { - System.out.println("testing shouldSetBits " + length); - int[] bitsToSet = createSortedIntArrayOfBitsToSet(length, 434222); - EWAHCompressedBitmap32 ewah = new EWAHCompressedBitmap32(); - System.out.println(" ... setting " + bitsToSet.length + " values"); - for (int i : bitsToSet) { - ewah.set(i); - } - System.out.println(" ... verifying " + bitsToSet.length + " values"); - equal(ewah.iterator(), bitsToSet); - System.out.println(" ... checking cardinality"); - Assert.assertEquals(bitsToSet.length, ewah.cardinality()); - } - - - @Test - public void testSizeInBits1() { - EWAHCompressedBitmap32 bitmap = new EWAHCompressedBitmap32(); - bitmap.setSizeInBits(1, false); - bitmap.not(); - Assert.assertEquals(1, bitmap.cardinality()); - } - - @Test - public void testHasNextSafe() { - EWAHCompressedBitmap32 bitmap = new EWAHCompressedBitmap32(); - bitmap.set(0); - IntIterator it = bitmap.intIterator(); - Assert.assertTrue(it.hasNext()); - Assert.assertEquals(0, it.next()); - } - - - @Test - public void testHasNextSafe2() { - EWAHCompressedBitmap32 bitmap = new EWAHCompressedBitmap32(); - bitmap.set(0); - IntIterator it = bitmap.intIterator(); - Assert.assertEquals(0, it.next()); - } - - @Test - public void testInfiniteLoop() { - System.out.println("Testing for an infinite loop"); + + /** + * a non-deterministic test proposed by Marc Polizzi. + * + * @param maxlength + * the maximum uncompressed size of the bitmap + */ + public static void PolizziTest(int maxlength) { + System.out.println("Polizzi test with max length = " + maxlength); + for (int k = 0; k < 10000; ++k) { + final Random rnd = new Random(); + final EWAHCompressedBitmap32 ewahBitmap1 = new EWAHCompressedBitmap32(); + final BitSet jdkBitmap1 = new BitSet(); + final EWAHCompressedBitmap32 ewahBitmap2 = new EWAHCompressedBitmap32(); + final BitSet jdkBitmap2 = new BitSet(); + final EWAHCompressedBitmap32 ewahBitmap3 = new EWAHCompressedBitmap32(); + final BitSet jdkBitmap3 = new BitSet(); + final int len = rnd.nextInt(maxlength); + for (int pos = 0; pos < len; pos++) { // random *** number of bits + // set *** + if (rnd.nextInt(7) == 0) { // random *** increasing *** values + ewahBitmap1.set(pos); + jdkBitmap1.set(pos); + } + if (rnd.nextInt(11) == 0) { // random *** increasing *** values + ewahBitmap2.set(pos); + jdkBitmap2.set(pos); + } + if (rnd.nextInt(7) == 0) { // random *** increasing *** values + ewahBitmap3.set(pos); + jdkBitmap3.set(pos); + } + } + assertEquals(jdkBitmap1, ewahBitmap1); + assertEquals(jdkBitmap2, ewahBitmap2); + assertEquals(jdkBitmap3, ewahBitmap3); + // XOR + { + final EWAHCompressedBitmap32 xorEwahBitmap = ewahBitmap1 + .xor(ewahBitmap2); + final BitSet xorJdkBitmap = (BitSet) jdkBitmap1.clone(); + xorJdkBitmap.xor(jdkBitmap2); + assertEquals(xorJdkBitmap, xorEwahBitmap); + } + // AND + { + final EWAHCompressedBitmap32 andEwahBitmap = ewahBitmap1 + .and(ewahBitmap2); + final BitSet andJdkBitmap = (BitSet) jdkBitmap1.clone(); + andJdkBitmap.and(jdkBitmap2); + assertEquals(andJdkBitmap, andEwahBitmap); + } + // AND + { + final EWAHCompressedBitmap32 andEwahBitmap = ewahBitmap2 + .and(ewahBitmap1); + final BitSet andJdkBitmap = (BitSet) jdkBitmap1.clone(); + andJdkBitmap.and(jdkBitmap2); + assertEquals(andJdkBitmap, andEwahBitmap); + assertEquals(andJdkBitmap, + EWAHCompressedBitmap32.and(ewahBitmap1, ewahBitmap2)); + } + // MULTI AND + { + final BitSet andJdkBitmap = (BitSet) jdkBitmap1.clone(); + andJdkBitmap.and(jdkBitmap2); + andJdkBitmap.and(jdkBitmap3); + assertEquals(andJdkBitmap, EWAHCompressedBitmap32.and( + ewahBitmap1, ewahBitmap2, ewahBitmap3)); + assertEquals(andJdkBitmap, EWAHCompressedBitmap32.and( + ewahBitmap3, ewahBitmap2, ewahBitmap1)); + Assert.assertEquals(andJdkBitmap.cardinality(), + EWAHCompressedBitmap32.andCardinality(ewahBitmap1, + ewahBitmap2, ewahBitmap3)); + } + // AND NOT + { + final EWAHCompressedBitmap32 andNotEwahBitmap = ewahBitmap1 + .andNot(ewahBitmap2); + final BitSet andNotJdkBitmap = (BitSet) jdkBitmap1.clone(); + andNotJdkBitmap.andNot(jdkBitmap2); + assertEquals(andNotJdkBitmap, andNotEwahBitmap); + } + // AND NOT + { + final EWAHCompressedBitmap32 andNotEwahBitmap = ewahBitmap2 + .andNot(ewahBitmap1); + final BitSet andNotJdkBitmap = (BitSet) jdkBitmap2.clone(); + andNotJdkBitmap.andNot(jdkBitmap1); + assertEquals(andNotJdkBitmap, andNotEwahBitmap); + } + // OR + { + final EWAHCompressedBitmap32 orEwahBitmap = ewahBitmap1 + .or(ewahBitmap2); + final BitSet orJdkBitmap = (BitSet) jdkBitmap1.clone(); + orJdkBitmap.or(jdkBitmap2); + assertEquals(orJdkBitmap, orEwahBitmap); + assertEquals(orJdkBitmap, + EWAHCompressedBitmap32.or(ewahBitmap1, ewahBitmap2)); + Assert.assertEquals(orEwahBitmap.cardinality(), + ewahBitmap1.orCardinality(ewahBitmap2)); + } + // OR + { + final EWAHCompressedBitmap32 orEwahBitmap = ewahBitmap2 + .or(ewahBitmap1); + final BitSet orJdkBitmap = (BitSet) jdkBitmap1.clone(); + orJdkBitmap.or(jdkBitmap2); + assertEquals(orJdkBitmap, orEwahBitmap); + } + // MULTI OR + { + final BitSet orJdkBitmap = (BitSet) jdkBitmap1.clone(); + orJdkBitmap.or(jdkBitmap2); + orJdkBitmap.or(jdkBitmap3); + assertEquals(orJdkBitmap, EWAHCompressedBitmap32.or( + ewahBitmap1, ewahBitmap2, ewahBitmap3)); + assertEquals(orJdkBitmap, EWAHCompressedBitmap32.or( + ewahBitmap3, ewahBitmap2, ewahBitmap1)); + Assert.assertEquals(orJdkBitmap.cardinality(), + EWAHCompressedBitmap32.orCardinality(ewahBitmap1, + ewahBitmap2, ewahBitmap3)); + } + } + } + + /** + * Pseudo-non-deterministic test inspired by Federico Fissore. + * + * @param length + * the number of set bits in a bitmap + */ + public static void shouldSetBits(int length) { + System.out.println("testing shouldSetBits " + length); + int[] bitsToSet = createSortedIntArrayOfBitsToSet(length, 434222); + EWAHCompressedBitmap32 ewah = new EWAHCompressedBitmap32(); + System.out.println(" ... setting " + bitsToSet.length + " values"); + for (int i : bitsToSet) { + ewah.set(i); + } + System.out.println(" ... verifying " + bitsToSet.length + " values"); + equal(ewah.iterator(), bitsToSet); + System.out.println(" ... checking cardinality"); + Assert.assertEquals(bitsToSet.length, ewah.cardinality()); + } + + @Test + public void testSizeInBits1() { + EWAHCompressedBitmap32 bitmap = new EWAHCompressedBitmap32(); + bitmap.setSizeInBits(1, false); + bitmap.not(); + Assert.assertEquals(1, bitmap.cardinality()); + } + + @Test + public void testHasNextSafe() { + EWAHCompressedBitmap32 bitmap = new EWAHCompressedBitmap32(); + bitmap.set(0); + IntIterator it = bitmap.intIterator(); + Assert.assertTrue(it.hasNext()); + Assert.assertEquals(0, it.next()); + } + + @Test + public void testHasNextSafe2() { + EWAHCompressedBitmap32 bitmap = new EWAHCompressedBitmap32(); + bitmap.set(0); + IntIterator it = bitmap.intIterator(); + Assert.assertEquals(0, it.next()); + } + + @Test + public void testInfiniteLoop() { + System.out.println("Testing for an infinite loop"); EWAHCompressedBitmap32 b1 = new EWAHCompressedBitmap32(); EWAHCompressedBitmap32 b2 = new EWAHCompressedBitmap32(); EWAHCompressedBitmap32 b3 = new EWAHCompressedBitmap32(); - b3.setSizeInBits(5); + b3.setSizeInBits(5,false); b1.set(2); b2.set(4); EWAHCompressedBitmap32.and(b1, b2, b3); EWAHCompressedBitmap32.or(b1, b2, b3); - } - - @Test - public void testSizeInBits2() { - EWAHCompressedBitmap32 bitmap = new EWAHCompressedBitmap32(); - bitmap.setSizeInBits(1, true); - bitmap.not(); - Assert.assertEquals(0, bitmap.cardinality()); - } - - private static void assertAndEquals(EWAHCompressedBitmap32... bitmaps) { - EWAHCompressedBitmap32 expected = bitmaps[0]; - for (int i = 1; i < bitmaps.length; i++) { - expected = expected.and(bitmaps[i]); - } - assertEquals(expected, EWAHCompressedBitmap32.and(bitmaps)); - } - - private static void assertEquals(EWAHCompressedBitmap32 expected, - EWAHCompressedBitmap32 actual) { - Assert.assertEquals(expected.sizeInBits(), actual.sizeInBits()); - assertEqualsPositions(expected, actual); - } - - private static void assertOrEquals(EWAHCompressedBitmap32... bitmaps) { - EWAHCompressedBitmap32 expected = bitmaps[0]; - for (int i = 1; i < bitmaps.length; i++) { - expected = expected.or(bitmaps[i]); - } - assertEquals(expected, EWAHCompressedBitmap32.or(bitmaps)); - } - - /** - * Extracted. - * - * @param bits - * the bits - * @return the integer - */ - private static Integer extracted(final Iterator bits) { - return bits.next(); - } - - private static void testSetSizeInBits(int size, int nextBit) { - EWAHCompressedBitmap32 bitmap = new EWAHCompressedBitmap32(); - bitmap.setSizeInBits(size, false); - bitmap.set(nextBit); - BitSet jdkBitmap = new BitSet(); - jdkBitmap.set(nextBit); - assertEquals(jdkBitmap, bitmap); - } - - /** - * Assess equality between an uncompressed bitmap and a compressed one, part - * of a test contributed by Marc Polizzi - * - * @param jdkBitmap - * the uncompressed bitmap - * @param ewahBitmap - * the compressed bitmap - */ - static void assertCardinality(BitSet jdkBitmap, - EWAHCompressedBitmap32 ewahBitmap) { - final int c1 = jdkBitmap.cardinality(); - final int c2 = ewahBitmap.cardinality(); - Assert.assertEquals(c1, c2); - } - - /** - * Assess equality between an uncompressed bitmap and a compressed one, part - * of a test contributed by Marc Polizzi. - * - * @param jdkBitmap - * the uncompressed bitmap - * @param ewahBitmap - * the compressed bitmap - */ - static void assertEquals(BitSet jdkBitmap, EWAHCompressedBitmap32 ewahBitmap) { - assertEqualsIterator(jdkBitmap, ewahBitmap); - assertEqualsPositions(jdkBitmap, ewahBitmap); - assertCardinality(jdkBitmap, ewahBitmap); - } - - static void assertEquals(int[] v, List p) { - assertEquals(p, v); - } - - static void assertEquals(List p, int[] v) { - if (v.length != p.size()) - throw new RuntimeException("Different lengths " + v.length + " " - + p.size()); - for (int k = 0; k < v.length; ++k) - if (v[k] != p.get(k).intValue()) - throw new RuntimeException("expected equal at " + k + " " + v[k] + " " - + p.get(k)); - } - - // - /** - * Assess equality between an uncompressed bitmap and a compressed one, part - * of a test contributed by Marc Polizzi - * - * @param jdkBitmap - * the jdk bitmap - * @param ewahBitmap - * the ewah bitmap - */ - static void assertEqualsIterator(BitSet jdkBitmap, - EWAHCompressedBitmap32 ewahBitmap) { - final Vector positions = new Vector(); - final Iterator bits = ewahBitmap.iterator(); - while (bits.hasNext()) { - final int bit = extracted(bits).intValue(); - Assert.assertTrue(jdkBitmap.get(bit)); - positions.add(new Integer(bit)); - } - for (int pos = jdkBitmap.nextSetBit(0); pos >= 0; pos = jdkBitmap - .nextSetBit(pos + 1)) { - if (!positions.contains(new Integer(pos))) { - throw new RuntimeException("iterator: bitset got different bits"); - } - } - } - - // part of a test contributed by Marc Polizzi - /** - * Assert equals positions. - * - * @param jdkBitmap - * the jdk bitmap - * @param ewahBitmap - * the ewah bitmap - */ - static void assertEqualsPositions(BitSet jdkBitmap, - EWAHCompressedBitmap32 ewahBitmap) { - final List positions = ewahBitmap.getPositions(); - for (int position : positions) { - if (!jdkBitmap.get(position)) { - throw new RuntimeException("positions: bitset got different bits"); - } - } - for (int pos = jdkBitmap.nextSetBit(0); pos >= 0; pos = jdkBitmap - .nextSetBit(pos + 1)) { - if (!positions.contains(new Integer(pos))) { - throw new RuntimeException("positions: bitset got different bits"); - } - } - // we check again - final int[] fastpositions = ewahBitmap.toArray(); - for (int position : fastpositions) { - if (!jdkBitmap.get(position)) { - throw new RuntimeException( - "positions: bitset got different bits with toArray"); - } - } - for (int pos = jdkBitmap.nextSetBit(0); pos >= 0; pos = jdkBitmap - .nextSetBit(pos + 1)) { - int index = Arrays.binarySearch(fastpositions, pos); - if (index < 0) - throw new RuntimeException( - "positions: bitset got different bits with toArray"); - if (fastpositions[index] != pos) - throw new RuntimeException( - "positions: bitset got different bits with toArray"); - } - } - - /** - * Assert equals positions. - * - * @param ewahBitmap1 - * the ewah bitmap1 - * @param ewahBitmap2 - * the ewah bitmap2 - */ - static void assertEqualsPositions(EWAHCompressedBitmap32 ewahBitmap1, - EWAHCompressedBitmap32 ewahBitmap2) { - final List positions1 = ewahBitmap1.getPositions(); - final List positions2 = ewahBitmap2.getPositions(); - if (!positions1.equals(positions2)) - throw new RuntimeException( - "positions: alternative got different bits (two bitmaps)"); - // - final int[] fastpositions1 = ewahBitmap1.toArray(); - assertEquals(fastpositions1, positions1); - final int[] fastpositions2 = ewahBitmap2.toArray(); - assertEquals(fastpositions2, positions2); - if (!Arrays.equals(fastpositions1, fastpositions2)) - throw new RuntimeException( - "positions: alternative got different bits with toArray but not with getPositions (two bitmaps)"); - } - - /** - * Convenience function to assess equality between a compressed bitset and an - * uncompressed bitset - * - * @param x - * the compressed bitset/bitmap - * @param y - * the uncompressed bitset/bitmap - */ - static void equal(EWAHCompressedBitmap32 x, BitSet y) { - Assert.assertEquals(x.cardinality(), y.cardinality()); - for (int i : x.getPositions()) - Assert.assertTrue(y.get(i)); - } - - /** - * Convenience function to assess equality between an array and an iterator - * over Integers - * - * @param i - * the iterator - * @param array - * the array - */ - static void equal(Iterator i, int[] array) { - int cursor = 0; - while (i.hasNext()) { - int x = extracted(i).intValue(); - int y = array[cursor++]; - Assert.assertEquals(x, y); - } - } + } + + @Test + public void testSizeInBits2() { + EWAHCompressedBitmap32 bitmap = new EWAHCompressedBitmap32(); + bitmap.setSizeInBits(1, true); + bitmap.not(); + Assert.assertEquals(0, bitmap.cardinality()); + } + + private static void assertAndEquals(EWAHCompressedBitmap32... bitmaps) { + EWAHCompressedBitmap32 expected = bitmaps[0]; + for (int i = 1; i < bitmaps.length; i++) { + expected = expected.and(bitmaps[i]); + } + Assert.assertTrue(expected.equals(EWAHCompressedBitmap32.and(bitmaps))); + } + + private static void assertEquals(EWAHCompressedBitmap32 expected, + EWAHCompressedBitmap32 actual) { + Assert.assertEquals(expected.sizeInBits(), actual.sizeInBits()); + assertEqualsPositions(expected, actual); + } + + private static void assertOrEquals(EWAHCompressedBitmap32... bitmaps) { + EWAHCompressedBitmap32 expected = bitmaps[0]; + for (int i = 1; i < bitmaps.length; i++) { + expected = expected.or(bitmaps[i]); + } + assertEquals(expected, EWAHCompressedBitmap32.or(bitmaps)); + } + + /** + * Extracted. + * + * @param bits + * the bits + * @return the integer + */ + private static Integer extracted(final Iterator bits) { + return bits.next(); + } + + private static void testSetSizeInBits(int size, int nextBit) { + EWAHCompressedBitmap32 bitmap = new EWAHCompressedBitmap32(); + bitmap.setSizeInBits(size, false); + bitmap.set(nextBit); + BitSet jdkBitmap = new BitSet(); + jdkBitmap.set(nextBit); + assertEquals(jdkBitmap, bitmap); + } + + /** + * Assess equality between an uncompressed bitmap and a compressed one, part + * of a test contributed by Marc Polizzi + * + * @param jdkBitmap + * the uncompressed bitmap + * @param ewahBitmap + * the compressed bitmap + */ + static void assertCardinality(BitSet jdkBitmap, + EWAHCompressedBitmap32 ewahBitmap) { + final int c1 = jdkBitmap.cardinality(); + final int c2 = ewahBitmap.cardinality(); + Assert.assertEquals(c1, c2); + } + + /** + * Assess equality between an uncompressed bitmap and a compressed one, part + * of a test contributed by Marc Polizzi. + * + * @param jdkBitmap + * the uncompressed bitmap + * @param ewahBitmap + * the compressed bitmap + */ + static void assertEquals(BitSet jdkBitmap, EWAHCompressedBitmap32 ewahBitmap) { + assertEqualsIterator(jdkBitmap, ewahBitmap); + assertEqualsPositions(jdkBitmap, ewahBitmap); + assertCardinality(jdkBitmap, ewahBitmap); + } + + static void assertEquals(int[] v, List p) { + assertEquals(p, v); + } + + static void assertEquals(List p, int[] v) { + if (v.length != p.size()) + throw new RuntimeException("Different lengths " + v.length + " " + + p.size()); + for (int k = 0; k < v.length; ++k) + if (v[k] != p.get(k).intValue()) + throw new RuntimeException("expected equal at " + k + " " + + v[k] + " " + p.get(k)); + } + + // + /** + * Assess equality between an uncompressed bitmap and a compressed one, part + * of a test contributed by Marc Polizzi + * + * @param jdkBitmap + * the jdk bitmap + * @param ewahBitmap + * the ewah bitmap + */ + static void assertEqualsIterator(BitSet jdkBitmap, + EWAHCompressedBitmap32 ewahBitmap) { + final Vector positions = new Vector(); + final Iterator bits = ewahBitmap.iterator(); + while (bits.hasNext()) { + final int bit = extracted(bits).intValue(); + Assert.assertTrue(jdkBitmap.get(bit)); + positions.add(new Integer(bit)); + } + for (int pos = jdkBitmap.nextSetBit(0); pos >= 0; pos = jdkBitmap + .nextSetBit(pos + 1)) { + if (!positions.contains(new Integer(pos))) { + throw new RuntimeException( + "iterator: bitset got different bits"); + } + } + } + + // part of a test contributed by Marc Polizzi + /** + * Assert equals positions. + * + * @param jdkBitmap + * the jdk bitmap + * @param ewahBitmap + * the ewah bitmap + */ + static void assertEqualsPositions(BitSet jdkBitmap, + EWAHCompressedBitmap32 ewahBitmap) { + final List positions = ewahBitmap.getPositions(); + for (int position : positions) { + if (!jdkBitmap.get(position)) { + throw new RuntimeException( + "positions: bitset got different bits"); + } + } + for (int pos = jdkBitmap.nextSetBit(0); pos >= 0; pos = jdkBitmap + .nextSetBit(pos + 1)) { + if (!positions.contains(new Integer(pos))) { + throw new RuntimeException( + "positions: bitset got different bits"); + } + } + // we check again + final int[] fastpositions = ewahBitmap.toArray(); + for (int position : fastpositions) { + if (!jdkBitmap.get(position)) { + throw new RuntimeException( + "positions: bitset got different bits with toArray"); + } + } + for (int pos = jdkBitmap.nextSetBit(0); pos >= 0; pos = jdkBitmap + .nextSetBit(pos + 1)) { + int index = Arrays.binarySearch(fastpositions, pos); + if (index < 0) + throw new RuntimeException( + "positions: bitset got different bits with toArray"); + if (fastpositions[index] != pos) + throw new RuntimeException( + "positions: bitset got different bits with toArray"); + } + } + + /** + * Assert equals positions. + * + * @param ewahBitmap1 + * the ewah bitmap1 + * @param ewahBitmap2 + * the ewah bitmap2 + */ + static void assertEqualsPositions(EWAHCompressedBitmap32 ewahBitmap1, + EWAHCompressedBitmap32 ewahBitmap2) { + final List positions1 = ewahBitmap1.getPositions(); + final List positions2 = ewahBitmap2.getPositions(); + if (!positions1.equals(positions2)) + throw new RuntimeException( + "positions: alternative got different bits (two bitmaps)"); + // + final int[] fastpositions1 = ewahBitmap1.toArray(); + assertEquals(fastpositions1, positions1); + final int[] fastpositions2 = ewahBitmap2.toArray(); + assertEquals(fastpositions2, positions2); + if (!Arrays.equals(fastpositions1, fastpositions2)) + throw new RuntimeException( + "positions: alternative got different bits with toArray but not with getPositions (two bitmaps)"); + } + + /** + * Convenience function to assess equality between a compressed bitset and + * an uncompressed bitset + * + * @param x + * the compressed bitset/bitmap + * @param y + * the uncompressed bitset/bitmap + */ + static void equal(EWAHCompressedBitmap32 x, BitSet y) { + Assert.assertEquals(x.cardinality(), y.cardinality()); + for (int i : x.getPositions()) + Assert.assertTrue(y.get(i)); + } + + /** + * Convenience function to assess equality between an array and an iterator + * over Integers + * + * @param i + * the iterator + * @param array + * the array + */ + static void equal(Iterator i, int[] array) { + int cursor = 0; + while (i.hasNext()) { + int x = extracted(i).intValue(); + int y = array[cursor++]; + Assert.assertEquals(x, y); + } + } - /** The Constant MEGA: a large integer. */ - private static final int MEGA = 8 * 1024 * 1024; + /** The Constant MEGA: a large integer. */ + private static final int MEGA = 8 * 1024 * 1024; - /** The Constant TEST_BS_SIZE: used to represent the size of a large bitmap. */ - private static final int TEST_BS_SIZE = 8 * MEGA; + /** The Constant TEST_BS_SIZE: used to represent the size of a large bitmap. */ + private static final int TEST_BS_SIZE = 8 * MEGA; } diff -Nru libjavaewah-java-0.6.12/src/test/java/com/googlecode/javaewah32/IntIteratorOverIteratingRLWTest32.java libjavaewah-java-0.7.9/src/test/java/com/googlecode/javaewah32/IntIteratorOverIteratingRLWTest32.java --- libjavaewah-java-0.6.12/src/test/java/com/googlecode/javaewah32/IntIteratorOverIteratingRLWTest32.java 1970-01-01 00:00:00.000000000 +0000 +++ libjavaewah-java-0.7.9/src/test/java/com/googlecode/javaewah32/IntIteratorOverIteratingRLWTest32.java 2013-11-12 14:31:20.000000000 +0000 @@ -0,0 +1,119 @@ +package com.googlecode.javaewah32; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +/* + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. + */ +/** + * Tests for utility class. Sketchy for now. + * + */ +@SuppressWarnings("javadoc") +public class IntIteratorOverIteratingRLWTest32 { + + @Test + // had problems with bitmaps beginning with two consecutive clean runs + public void testConsecClean() { + System.out + .println("testing int iteration, 2 consec clean runs starting with zeros"); + EWAHCompressedBitmap32 e = new EWAHCompressedBitmap32(); + for (int i = 64; i < 128; ++i) + e.set(i); + IntIteratorOverIteratingRLW32 ii = new IntIteratorOverIteratingRLW32( + e.getIteratingRLW()); + assertTrue(ii.hasNext()); + int ctr = 0; + while (ii.hasNext()) { + ++ctr; + ii.next(); + } + assertEquals(64, ctr); + } + + @Test + public void testConsecCleanStartOnes() { + System.out + .println("testing int iteration, 2 consec clean runs starting with ones"); + EWAHCompressedBitmap32 e = new EWAHCompressedBitmap32(); + for (int i = 0; i < 2 * 64; ++i) + e.set(i); + for (int i = 4 * 64; i < 5 * 64; ++i) + e.set(i); + + IntIteratorOverIteratingRLW32 ii = new IntIteratorOverIteratingRLW32( + e.getIteratingRLW()); + assertTrue(ii.hasNext()); + int ctr = 0; + while (ii.hasNext()) { + ++ctr; + ii.next(); + } + assertEquals(3 * 64, ctr); + } + + @Test + public void testStartDirty() { + System.out.println("testing int iteration, no initial runs"); + EWAHCompressedBitmap32 e = new EWAHCompressedBitmap32(); + for (int i = 1; i < 2 * 64; ++i) + e.set(i); + for (int i = 4 * 64; i < 5 * 64; ++i) + e.set(i); + + IntIteratorOverIteratingRLW32 ii = new IntIteratorOverIteratingRLW32( + e.getIteratingRLW()); + assertTrue(ii.hasNext()); + int ctr = 0; + while (ii.hasNext()) { + ++ctr; + ii.next(); + } + assertEquals(3 * 64 - 1, ctr); + } + + @Test + public void testEmpty() { + System.out.println("testing int iteration over empty bitmap"); + EWAHCompressedBitmap32 e = new EWAHCompressedBitmap32(); + + IntIteratorOverIteratingRLW32 ii = new IntIteratorOverIteratingRLW32( + e.getIteratingRLW()); + assertFalse(ii.hasNext()); + } + + @Test + public void testRandomish() { + EWAHCompressedBitmap32 e = new EWAHCompressedBitmap32(); + + int upperlimit = 100000; + for (int i = 0; i < upperlimit; ++i) { + double probabilityOfOne = i / (double) (upperlimit / 2); + if (probabilityOfOne > 1.0) + probabilityOfOne = 1.0; + if (Math.random() < probabilityOfOne) { + e.set(i); + } + } + + IntIteratorOverIteratingRLW32 ii = new IntIteratorOverIteratingRLW32( + e.getIteratingRLW()); + int ctr = 0; + while (ii.hasNext()) { + ++ctr; + ii.next(); + } + + assertEquals(e.cardinality(), ctr); + System.out + .println("checking int iteration over a var density bitset of size " + + e.cardinality()); + + } + +} diff -Nru libjavaewah-java-0.6.12/src/test/java/com/googlecode/javaewah32/IteratorAggregationTest32.java libjavaewah-java-0.7.9/src/test/java/com/googlecode/javaewah32/IteratorAggregationTest32.java --- libjavaewah-java-0.6.12/src/test/java/com/googlecode/javaewah32/IteratorAggregationTest32.java 1970-01-01 00:00:00.000000000 +0000 +++ libjavaewah-java-0.7.9/src/test/java/com/googlecode/javaewah32/IteratorAggregationTest32.java 2013-11-12 14:31:20.000000000 +0000 @@ -0,0 +1,145 @@ +package com.googlecode.javaewah32; + +import static org.junit.Assert.*; +import java.util.Iterator; +import org.junit.Test; +import com.googlecode.javaewah.benchmark.ClusteredDataGenerator; + +/* + * Copyright 2009-2013, Daniel Lemire, Cliff Moon, David McIntosh, Robert Becho, Google Inc., Veronika Zenz and Owen Kaser + * Licensed under the Apache License, Version 2.0. + */ +/** + * Tests specifically for iterators. + * + */ +public class IteratorAggregationTest32 { + + /** + * @param N number of bitmaps to generate in each set + * @param nbr parameter determining the size of the arrays (in a log scale) + * @return an iterator over sets of bitmaps + */ + public static Iterator getCollections(final int N, final int nbr) { + final ClusteredDataGenerator cdg = new ClusteredDataGenerator(123); + return new Iterator() { + int sparsity = 1; + + @Override + public boolean hasNext() { + return this.sparsity < 5; + } + + @Override + public EWAHCompressedBitmap32[] next() { + int[][] data = new int[N][]; + int Max = (1 << (nbr + this.sparsity)); + for (int k = 0; k < N; ++k) + data[k] = cdg.generateClustered(1 << nbr, Max); + EWAHCompressedBitmap32[] ewah = new EWAHCompressedBitmap32[N]; + for (int k = 0; k < N; ++k) { + ewah[k] = new EWAHCompressedBitmap32(); + for (int x = 0; x < data[k].length; ++x) { + ewah[k].set(data[k][x]); + } + data[k] = null; + } + this.sparsity += 3; + return ewah; + } + + @Override + public void remove() { + // unimplemented + } + + }; + + } + + /** + * + */ + @Test + public void testAnd() { + for (int N = 1; N < 10; ++N) { + System.out.println("testAnd N = " + N); + Iterator i = getCollections(N,3); + while (i.hasNext()) { + EWAHCompressedBitmap32[] x = i.next(); + EWAHCompressedBitmap32 tanswer = EWAHCompressedBitmap32.and(x); + EWAHCompressedBitmap32 x1 = IteratorUtil32 + .materialize(IteratorAggregation32.bufferedand(IteratorUtil32 + .toIterators(x))); + assertTrue(x1.equals(tanswer)); + } + System.gc(); + } + + } + + /** + * + */ + @Test + public void testOr() { + for (int N = 1; N < 10; ++N) { + System.out.println("testOr N = " + N); + Iterator i = getCollections(N,3); + while (i.hasNext()) { + EWAHCompressedBitmap32[] x = i.next(); + EWAHCompressedBitmap32 tanswer = EWAHCompressedBitmap32.or(x); + EWAHCompressedBitmap32 x1 = IteratorUtil32 + .materialize(IteratorAggregation32.bufferedor(IteratorUtil32 + .toIterators(x))); + assertTrue(x1.equals(tanswer)); + } + System.gc(); + } + } + + + /** + * + */ + @SuppressWarnings("deprecation") + @Test + public void testWideOr() { + for (int nbr = 3; nbr <= 24; nbr += 3) { + for (int N = 100; N < 1000; N += 100) { + System.out.println("testWideOr N = " + N); + Iterator i = getCollections(N, 3); + while (i.hasNext()) { + EWAHCompressedBitmap32[] x = i.next(); + EWAHCompressedBitmap32 tanswer = EWAHCompressedBitmap32.or(x); + EWAHCompressedBitmap32 container = new EWAHCompressedBitmap32(); + FastAggregation32.legacy_orWithContainer(container, x); + assertTrue(container.equals(tanswer)); + EWAHCompressedBitmap32 x1 = IteratorUtil32 + .materialize(IteratorAggregation32 + .bufferedor(IteratorUtil32.toIterators(x))); + assertTrue(x1.equals(tanswer)); + } + System.gc(); + } + } + } + /** + * + */ + @Test + public void testXor() { + System.out.println("testXor "); + Iterator i = getCollections(2,3); + while (i.hasNext()) { + EWAHCompressedBitmap32[] x = i.next(); + EWAHCompressedBitmap32 tanswer = x[0].xor(x[1]); + EWAHCompressedBitmap32 x1 = IteratorUtil32 + .materialize(IteratorAggregation32.bufferedxor( + x[0].getIteratingRLW(), x[1].getIteratingRLW())); + assertTrue(x1.equals(tanswer)); + } + System.gc(); + } + +} diff -Nru libjavaewah-java-0.6.12/.travis.yml libjavaewah-java-0.7.9/.travis.yml --- libjavaewah-java-0.6.12/.travis.yml 1970-01-01 00:00:00.000000000 +0000 +++ libjavaewah-java-0.7.9/.travis.yml 2013-11-12 14:31:20.000000000 +0000 @@ -0,0 +1,10 @@ +language: java + +jdk: + - oraclejdk7 + - openjdk7 + - openjdk6 + +install: true + +script: mvn test