[c++11/14/17] Biblioteka do przechowywania danych więcej niż może ram

import java.io.*;
import java.util.ArrayList;
import java.util.Arrays;

public class ChunkedArray<Type> {


    public static class CacheError extends Exception {

        private String msg;

        public CacheError(String msg) {
            this.msg = msg;
        }

        public String toString() {
            return this.msg;
        }
    }

    //-----------------------------------------------------------



    public  Object[]  currentCache;
    private Type      defaultValue;
    private String    cacheDir;
    private long      maxSize;
    private int       chunkSize;
    private int       chunkPosition;
    private long      chunkCount;
    private long      currentIndex;
    private long      cacheFile;


    //-----------------------------------------------------------



    private ChunkedArray(String cacheDir)
            throws ChunkedArray.CacheError
    {
        //throw new ChunkedArray.CacheError("Functionality never coded before");

        try {
            FileInputStream fstream = new FileInputStream(cacheDir + "/cache_info.txt");
            BufferedReader br = new BufferedReader(new InputStreamReader(fstream));
            String strLine = "";
            ArrayList<String> cacheInfo = new ArrayList<>();

            while ((strLine = br.readLine()) != null)
                cacheInfo.add(strLine);

            fstream.close();
            br.close();


            this.maxSize = Long.valueOf( cacheInfo.get(0).split(":")[1] );
            this.chunkSize = Integer.valueOf( cacheInfo.get(1).split(":")[1] );
            String fileDefaultValue = cacheInfo.get(2).split(":")[1];


            System.out.println(">>fis="+cacheDir + "/" + fileDefaultValue.replaceAll("\n",""));
            fstream = new FileInputStream(cacheDir + "/" + fileDefaultValue.replaceAll("\n",""));
            ObjectInputStream ois = new ObjectInputStream(fstream);
            this.defaultValue = (Type) ois.readObject(); //type unsafe but works

            System.out.println("this.defV: " + this.defaultValue);

            this.currentCache = new Object[this.chunkSize];


            this.loadFromDisk(0);
            this.currentIndex = -1;
            this.chunkPosition = -1;
            this.cacheDir = cacheDir;

            System.out.println("zaladowano!");

        } catch (Exception e) {
            throw new ChunkedArray.CacheError("ChunkedArray ctor CacheError: "+e.getClass().getName());
        }

    }


    public ChunkedArray(Type defaulValue, String cacheDir, long maxSize, int chunkSize)
        throws ChunkedArray.CacheError
    {
        this.defaultValue = defaulValue;
        this.cacheDir = cacheDir;
        this.maxSize = maxSize;
        this.chunkSize = chunkSize;

        this.chunkPosition = 0;
        this.chunkCount = (int)( maxSize / chunkSize) + 1;

        this.currentIndex = 0;

        this.resetCache();
        this.fillCacheDefaultValues();

        this.createDiskStructure();
        this.saveCacheInfo();
        this.setCacheFile(this.currentIndex);
        this.saveCacheInfo();
    }



    private void createDiskStructure() {
        long chunkCount = (long) this.maxSize / this.chunkSize;
        (new File("./" + this.cacheDir)).mkdir();
        for (int i = 0; i < this.chunkCount; i++) {
            try {
                //(new File("./" + this.cacheDir + "/" + i + ".txt")).createNewFile();
                FileOutputStream fos = new FileOutputStream("./" + this.cacheDir + "/" + i + ".txt");
                ObjectOutputStream oos = new ObjectOutputStream(fos);
                oos.writeObject(this.currentCache);
                oos.close();
                fos.close();

            } catch (Exception e) {
                System.out.println("Create disk structure info");
            }
        }
    }


    public void fillCacheDefaultValues() {
        for (int i=0;  i<this.chunkSize;  i++)
            this.currentCache[i] = this.defaultValue;
    }


    public void resetCache() {
        this.currentCache = new Object[this.chunkSize];

        for (int i=0;  i<this.chunkSize;  i++)
            this.currentCache[i] = this.defaultValue;
    }


    public void saveCacheInfo() throws ChunkedArray.CacheError {
        try {
            PrintWriter pw = new PrintWriter("./" + this.cacheDir + "/cache_info.txt");
            pw.write("maxSize:" + this.maxSize + "\n");
            pw.write("chunkSize:" + this.chunkSize + "\n");
            pw.write("defaultValueFile:defaultValue.txt\n");
            pw.close();

            FileOutputStream fos = new FileOutputStream("./" + this.cacheDir + "/defaultValue.txt");
            ObjectOutputStream oos = new ObjectOutputStream(fos);

            oos.writeObject(this.defaultValue);
            oos.close();
            fos.close();

        } catch (Exception e) {
            throw new ChunkedArray.CacheError("ChunkedArray.CacheError: saveCacheInfo Error");
        }
    }


    public void setCacheFile(long cacheFile) {
        this.cacheFile = cacheFile;
    }


    public ChunkedArray<Type> set(long index, Type value) throws ChunkedArray.CacheError
    {
        if (index > this.maxSize)
            throw new ChunkedArray.CacheError("Index out of bounds exception (index > maxSize");
        if (index < 0)
            throw new ChunkedArray.CacheError("Index out of bounds exception (index < 0");

        this.selectCacheFile(index);
        this.currentCache[ (int) (index % this.chunkSize) ] = value;
        return this;
    }


    public Type get(long index) throws ChunkedArray.CacheError {

        if (index > this.maxSize)
            throw new ChunkedArray.CacheError("Index out of bounds exception (index > maxSize");
        if (index < 0)
            throw new ChunkedArray.CacheError("Index out of bounds exception (index < 0");

        this.selectCacheFile(index);
        return (Type) this.currentCache[ (int) (index % this.chunkSize) ];
    }


    public void saveCache() {
        this.saveToDisk(this.currentIndex);
    }


    public void saveToDisk(long fileIndex) {
        try {
            FileOutputStream fos = new FileOutputStream("./" + this.cacheDir + "/" + fileIndex + ".txt");
            ObjectOutputStream oos = new ObjectOutputStream(fos);
            oos.writeObject(this.currentCache);
        } catch (Exception e) {
            System.out.println("ChunkedArray.saveToDisk() error: " + e);
            e.printStackTrace();
        }
    }


    public void loadFromDisk(long fileIndex) {
        if (fileIndex == this.currentIndex) return;

        try {
            //System.out.println("Loading cache...");
            FileInputStream fis = new FileInputStream("./" + this.cacheDir + "/" + fileIndex + ".txt"); //new FileInputStream("student.ser");
            ObjectInputStream ois = new ObjectInputStream(fis);
            this.currentCache = (Type[]) ois.readObject();

            this.currentIndex = fileIndex;
        } catch (Exception e) {
            System.out.println("ChunkedArray.loadFromDisk() error at index: " + fileIndex + ", with error:"+ e);
            e.printStackTrace();
        }
    }


    public void selectCacheFile(long index) throws ChunkedArray.CacheError
    {
        long newIndex = 0;

        if (index != 0)
            newIndex = index / this.chunkSize;

        if (newIndex != this.currentIndex) {
            System.out.println("selectCacheFile: swap cache needed");
            this.saveToDisk(this.currentIndex);
            this.loadFromDisk(newIndex);
            this.currentIndex = newIndex;
        }
    }



    public long getMaxSize() {
        return this.maxSize;
    }


    //@Override
    public String toStringXY() {
        return "ChunkedArray{" +
                "currentCache=" + /*Arrays.toString(currentCache)*/  currentCache.length +
                ", defaultValue=" + defaultValue +
                ", cacheDir='" + cacheDir + '\n' +
                ", maxSize=" + maxSize + '\n' +
                ", chunkSize=" + chunkSize + '\n' +
                ", chunkPosition=" + chunkPosition + '\n' +
                ", chunkCount=" + chunkCount + '\n' +
                ", currentIndex=" + currentIndex + '\n' +
                ", cacheFile=" + cacheFile + '\n' +
                '}';
    }

    @Override
    public String toString() {

        /*

    public  Object[]  currentCache;
    private Type      defaultValue;
    private String    cacheDir;
    private long      maxSize;
    private int       chunkSize;
    private int       chunkPosition;
    private long      chunkCount;
    private long      currentIndex;
    private long      cacheFile;

         */

        StringBuilder sb = new StringBuilder();
        sb.append("currentCache.length: " + currentCache.length+"\n");
        sb.append("defaultType: " + defaultValue+"\n");
        sb.append("cacheDir: " + cacheDir+"\n");
        sb.append("maxSize:" + maxSize+"\n");
        sb.append("chunkSize: " + chunkSize+"\n");
        sb.append("chunkPosition: + " + chunkPosition+"\n");
        sb.append("chunkCount: " + chunkCount+"\n");
        sb.append("currentIndex: " + currentIndex+"\n");
        sb.append("cacheFile: " + cacheFile+"\n");

        return sb.toString();
    }

    public static <X> ChunkedArray<X>  loadFromCache(String cacheDir) throws ChunkedArray.CacheError
    {
        return new ChunkedArray<X>(cacheDir);
    }
}

import java.io.*;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Iterator;

//https://stackoverflow.com/questions/5849154/can-we-write-our-own-iterator-in-java
//https://www.youtube.com/watch?v=tIOBLlehkpA




public class BoolArray /*implements Iterable<boolean>*/ {


    public class CacheError extends Exception {

        private String msg;

        public CacheError(String msg) {
            this.msg = msg;
        }

        public String toString() {
            return this.msg;
        }
    }


    private int chunkSize;
    //private BigInteger maxSize;
    private BigInteger maxSize;
    private boolean defaultValue;
    private BigInteger currentIndex;
    private String cacheDir;

    private BigInteger chunkSize_;
    private BigInteger position;
    private int chunkPosition, currentChunk, chunkCount;

    private boolean []currentCache;


    private BoolArray(String cacheDir) throws CacheError {
        try {
            FileInputStream fstream = new FileInputStream(cacheDir + "/cache_info.txt");
            BufferedReader br = new BufferedReader(new InputStreamReader(fstream));
            String strLine;
            ArrayList<String> cacheInfo = new ArrayList<>();

            while ((strLine = br.readLine()) != null)
                cacheInfo.add(strLine);

            br.close();

            this.maxSize = new BigInteger( cacheInfo.get(0).split(":")[1] );
            //this.chunkSize = new BigInteger( cacheInfo.get(1).split(":")[1] );
            this.chunkSize = Long.valueOf( cacheInfo.get(1).split(":")[1] ).intValue();
            this.defaultValue = Boolean.valueOf( cacheInfo.get(2).split(":")[1] );
            this.currentIndex = BigInteger.ZERO;
            this.cacheDir = cacheDir;

            this.loadFromDisk(this.currentIndex);

            this.chunkSize_ = new BigInteger(this.chunkSize+"");
            this.chunkPosition = 0;
            this.chunkCount = maxSize.divide(this.chunkSize_).intValue();

            System.out.println("BoolArray ctor ends");

        }catch (Exception e) {
            throw new CacheError("Something went wrong qrde");
        }

    }



    public BoolArray(BigInteger maxSize, int chunkSize, String cacheDir, boolean defaultValue)
            throws BoolArray.CacheError
    {
        this.maxSize = maxSize;
        this.chunkSize = chunkSize;
        this.cacheDir = cacheDir;
        this.defaultValue = defaultValue;

        this.resetCache();
        this.fillCacheDefaultVaues();

        this.createDiskStructure();
        this.saveCacheInfo();

        this.currentIndex = BigInteger.ZERO;
        this.selectCacheFile(this.currentIndex);

        this.chunkSize_ = new BigInteger(this.chunkSize+"");
        this.chunkPosition = 0;
        this.chunkCount = maxSize.divide(this.chunkSize_).intValue();

        System.out.println("BoolArray ctor ends");
    }


    private void fillCacheDefaultVaues() {
        for (int i=0;  i<chunkSize;  i++)
            this.currentCache[i] = defaultValue;

        System.out.println("fillCacheDefaultValus ends");
    }


    private void resetCache() {
        this.currentCache = new boolean[chunkSize/*.intValue()*/];
    }


    public void saveCacheInfo() throws BoolArray.CacheError {
        try {
            PrintWriter pw = new PrintWriter("./" + this.cacheDir + "/cache_info.txt");
            pw.write("maxSize:" + this.maxSize + "\n");
            pw.write("chunkSize:" + this.chunkSize + "\n");
            pw.write("defaultValue:" + this.defaultValue + "\n");
            pw.close();
        } catch (Exception e) {
            throw new CacheError("BoolArray.saveCacheInfo exception with cache: " + this.cacheDir);
        }
    }


    public void createDiskStructure() throws CacheError {
        BigInteger chunk_count = this.maxSize.divide(new BigInteger(this.chunkSize+"")).add(BigInteger.ONE);

        (new File("./" + this.cacheDir)).mkdir();

        for (BigInteger i = BigInteger.ZERO; i.compareTo(chunk_count) < 0; i = i.add(BigInteger.ONE)) {
            try {

                //(new File("./" + this.cacheDir + "/" + i + ".txt")).createNewFile();

                this.saveToDisk(i);

            } catch (Exception e) {
                throw new CacheError("BoolArray.createDiscStructure exception in for-loop with i="+i);
            }
        }
        System.out.println("createDiskStructure ends");
    }


    public void selectCacheFile(BigInteger index) throws CacheError {

        BigInteger newIndex = BigInteger.ZERO;

        if (index.compareTo(BigInteger.ZERO) != 0)
            //newIndex = (int) (this.chunkSize / index);
            newIndex = index.divide(this.chunkSize_);

        if (newIndex.compareTo(this.currentIndex)!=0) {
            //System.out.println("cache file index change! new="+newIndex+", prev="+this.currentIndex);
            System.out.println("selectCacheFile: swap cache");
            this.saveToDisk(this.currentIndex);
            this.loadFromDisk(newIndex);
            this.currentIndex = newIndex;
        }
        //System.out.println("selectCacheFile ends");
    }


    public void set(BigInteger index, boolean value) throws CacheError {
        //System.out.println("set index:"+index);

        this.selectCacheFile(index);

        BigInteger mod = index.mod(this.chunkSize_);

        this.currentCache[ mod.intValue() ] = value;
    }



    public boolean get(BigInteger index) throws CacheError {
        selectCacheFile(index);

        return this.currentCache[ index.mod(this.chunkSize_).intValue() ] ;
    }


    public boolean[]  getCurrentCache() {
        return currentCache;
    }


    public boolean[]  loadCacheFromFile(String path)
        throws BoolArray.CacheError
    {
        try {
            //System.out.println("Loading cache...");
            FileInputStream fis = new FileInputStream(path); //new FileInputStream("student.ser");
            ObjectInputStream ois = new ObjectInputStream(fis);
            return (boolean[]) ois.readObject();
        } catch (Exception e) {
            throw new CacheError("BoolArray.loadCacheFromFile"+path);
        }
    }


    public boolean setCurrentCache(boolean []newCache) throws BoolArray.CacheError {
        if (newCache.length != this.currentCache.length)
            throw new BoolArray.CacheError("Cannot setCurrentCache");

        this.currentCache = newCache;
        return true;
    }



    public void saveToDisk(BigInteger fileIndex) throws CacheError {

        try {
            FileOutputStream fos = new FileOutputStream("./" + this.cacheDir + "/" + fileIndex + ".txt");
            ObjectOutputStream oos = new ObjectOutputStream(fos);
            oos.writeObject(this.currentCache);
        } catch (Exception e) {
            throw new CacheError("BoolArray.saveToDisk error");
        }
        /*try {
            PrintWriter out = new PrintWriter(this.cacheDir + "/" + fileIndex +".txt");
            for (int i=0;  i<this.currentCache.length;  i++)
                out.write(this.currentCache[i]+"");
        } catch (Exception e) {
            throw new CacheError("BoolArray.saveToDisk exception");
        }*/
        System.out.println("saveToDisk ends");
    }


    public void saveCache() throws CacheError {
        this.saveToDisk(this.currentIndex);
        System.out.println("saveCache");
    }


    public void loadFromDisk(BigInteger fileIndex) throws BoolArray.CacheError {

        /*this.currentCache = FilesUtils.loadFromFile(this.cacheDir + "/" + fileIndex + ".txt");
        this.currentIndex = fileIndex;*/


        if (fileIndex.compareTo(this.currentIndex) ==0 ) return;

        String path = "./" + this.cacheDir + "/" + fileIndex + ".txt";

        new File(path).exists();


        //size inefficient
        try {
            //System.out.println("Loading cache...");

            FileInputStream fis = new FileInputStream(path); //new FileInputStream("student.ser");
            ObjectInputStream ois = new ObjectInputStream(fis);
            this.currentCache = (boolean[]) ois.readObject();

            this.currentIndex = fileIndex;
            System.out.println("loadFromDisk ends");
        } catch (Exception e) {
            throw new CacheError("BoolArray.loadFromDisk exception with fileindex="+fileIndex+", and path="+path);
        }

        //time inefficient?
        /*
        try {
            FileReader fr = null;
            BufferedReader r = new BufferedReader(new FileReader(this.cacheDir+"/"+fileIndex+".txt"));
            int ch;

            this.resetCache();
            int pos = -1;
            while ((ch = r.read()) != -1) {
                pos++;
                if (ch=='1')
                    this.currentCache[pos] = true;
                else
                    this.currentCache[pos] = false;
            }
        } catch (Exception e) {
            throw new CacheError("BoolArray.loadFromDisk");
        }*/
    }


    public BigInteger getMaxSize() {
        return this.maxSize;
    }


    public static BoolArray  loadFromCache(String cacheDir) throws CacheError {
        return new BoolArray(cacheDir);
    }


    public String toString() {

        StringBuilder sb = new StringBuilder();
        sb.append("BoolArray info:\n\n");
        sb.append("chunkSize:    " + chunkSize + "\n");
        sb.append("maxSize:      " + maxSize + "\n");
        sb.append("defaultValue: " + defaultValue + "\n");
        sb.append("currentIndex  " + currentIndex + "\n");
        sb.append("cacheDir:     " + cacheDir+"\n\n");

        return sb.toString();
    }


    //----------


    public boolean getFirst() throws BoolArray.CacheError {
        this.chunkPosition = 0;
        this.currentChunk = 0;
        return this.get(BigInteger.ZERO);
    }


    public boolean getNext() throws BoolArray.CacheError {
        this.chunkPosition++;
        if (this.chunkPosition > this.chunkSize) {
            this.chunkPosition = 0;
            this.currentChunk++;

            if (this.currentChunk > this.chunkCount)
                throw new CacheError("BoolArray.getNext has reached end of collction");

            this.loadFromDisk(BigInteger.valueOf(this.currentChunk));
            return this.currentCache[0];
        }
        return this.currentCache[this.chunkPosition];
    }

    public boolean getLast() throws BoolArray.CacheError {
        this.chunkPosition = this.chunkSize;
        this.currentChunk  = this.chunkCount;
        this.position = this.maxSize;
        return this.get(this.maxSize);
    }

    public boolean setPosition(BigInteger position) throws BoolArray.CacheError {
        this.get(position);

        this.position = position;
        return false;
    }

    public boolean hasNext() throws BoolArray.CacheError {
        return this.position.add(BigInteger.ONE).compareTo(this.maxSize) < 0;

    }
}